A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[core] platform agnostic executor via collective_rpc (#11256)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Jan 15, 2025 at 13:45 UTC
ad34c0df0f1b26b303a590133685b29e3daad20e
43 files changed
+852
-2642
tests/engine/test_custom_executor.py
+12
-16
@@ -1,12 +1,13 @@
1
import asyncio
2
import os
3
+from typing import Any, Dict, List, Optional, Tuple
4
5
import pytest
6
7
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
8
from vllm.engine.async_llm_engine import AsyncLLMEngine
9
from vllm.engine.llm_engine import LLMEngine
9
-from vllm.executor.gpu_executor import GPUExecutor, GPUExecutorAsync
10
+from vllm.executor.uniproc_executor import UniProcExecutor
11
from vllm.sampling_params import SamplingParams
12
13
@@ -14,21 +15,20 @@ class Mock:
15
...
16
17
17
-class CustomGPUExecutor(GPUExecutor):
18
+class CustomUniExecutor(UniProcExecutor):
19
19
- def execute_model(self, *args, **kwargs):
20
+ def collective_rpc(self,
21
+ method: str,
22
+ timeout: Optional[float] = None,
23
+ args: Tuple = (),
24
+ kwargs: Optional[Dict] = None) -> List[Any]:
25
# Drop marker to show that this was ran
26
with open(".marker", "w"):
27
...
23
- return super().execute_model(*args, **kwargs)
28
+ return super().collective_rpc(method, timeout, args, kwargs)
29
30
26
-class CustomGPUExecutorAsync(GPUExecutorAsync):
27
-
28
- async def execute_model_async(self, *args, **kwargs):
29
- with open(".marker", "w"):
30
- ...
31
- return await super().execute_model_async(*args, **kwargs)
31
+CustomUniExecutorAsync = CustomUniExecutor
32
33
34
@pytest.mark.parametrize("model", ["facebook/opt-125m"])
@@ -41,10 +41,6 @@ def test_custom_executor_type_checking(model):
41
engine_args = AsyncEngineArgs(model=model,
42
distributed_executor_backend=Mock)
43
AsyncLLMEngine.from_engine_args(engine_args)
44
- with pytest.raises(TypeError):
45
- engine_args = AsyncEngineArgs(
46
- model=model, distributed_executor_backend=CustomGPUExecutor)
47
- AsyncLLMEngine.from_engine_args(engine_args)
44
45
46
@pytest.mark.parametrize("model", ["facebook/opt-125m"])
@@ -55,7 +51,7 @@ def test_custom_executor(model, tmp_path):
51
assert not os.path.exists(".marker")
52
53
engine_args = EngineArgs(
58
- model=model, distributed_executor_backend=CustomGPUExecutor)
54
+ model=model, distributed_executor_backend=CustomUniExecutor)
55
engine = LLMEngine.from_engine_args(engine_args)
56
sampling_params = SamplingParams(max_tokens=1)
57
@@ -75,7 +71,7 @@ def test_custom_executor_async(model, tmp_path):
71
assert not os.path.exists(".marker")
72
73
engine_args = AsyncEngineArgs(
78
- model=model, distributed_executor_backend=CustomGPUExecutorAsync)
74
+ model=model, distributed_executor_backend=CustomUniExecutorAsync)
75
engine = AsyncLLMEngine.from_engine_args(engine_args)
76
sampling_params = SamplingParams(max_tokens=1)
77
tests/engine/test_multiproc_workers.py
+6
-6
@@ -6,16 +6,15 @@ from typing import Any, List, Tuple
6
7
import pytest
8
9
+from vllm.config import VllmConfig
10
from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
11
ResultHandler, WorkerMonitor)
12
+from vllm.worker.worker_base import WorkerWrapperBase
13
14
13
-class DummyWorker:
15
+class DummyWorkerWrapper(WorkerWrapperBase):
16
"""Dummy version of vllm.worker.worker.Worker"""
17
16
- def __init__(self, rank: int):
17
- self.rank = rank
18
-
18
def worker_method(self, worker_input: Any) -> Tuple[int, Any]:
19
sleep(0.05)
20
@@ -28,9 +27,10 @@ class DummyWorker:
27
28
def _start_workers() -> Tuple[List[ProcessWorkerWrapper], WorkerMonitor]:
29
result_handler = ResultHandler()
30
+ vllm_config = VllmConfig()
31
workers = [
32
- ProcessWorkerWrapper(result_handler, partial(DummyWorker, rank=rank))
33
- for rank in range(8)
32
+ ProcessWorkerWrapper(result_handler, DummyWorkerWrapper, vllm_config,
33
+ rank) for rank in range(8)
34
]
35
36
worker_monitor = WorkerMonitor(workers, result_handler)
tests/test_utils.py
+5
-1
@@ -2,6 +2,7 @@ import asyncio
2
import os
3
import socket
4
from typing import AsyncIterator, Tuple
5
+from unittest.mock import patch
6
7
import pytest
8
import torch
@@ -390,7 +391,10 @@ def test_bind_kv_cache_encoder_decoder():
391
392
393
def test_bind_kv_cache_pp():
393
- cfg = VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=2))
394
+ with patch("vllm.utils.cuda_device_count_stateless", lambda: 2):
395
+ # this test runs with 1 GPU, but we simulate 2 GPUs
396
+ cfg = VllmConfig(
397
+ parallel_config=ParallelConfig(pipeline_parallel_size=2))
398
with set_current_vllm_config(cfg):
399
from vllm.attention import Attention
400
vllm/config.py
+8
-4
@@ -1294,8 +1294,11 @@ class ParallelConfig:
1294
from vllm.executor import ray_utils
1295
backend = "mp"
1296
ray_found = ray_utils.ray_is_available()
1297
- if (current_platform.is_cuda()
1298
- and cuda_device_count_stateless() < self.world_size):
1297
+ if current_platform.is_neuron():
1298
+ # neuron uses single process to control multiple devices
1299
+ backend = "uni"
1300
+ elif (current_platform.is_cuda()
1301
+ and cuda_device_count_stateless() < self.world_size):
1302
if not ray_found:
1303
raise ValueError("Unable to load Ray which is "
1304
"required for multi-node inference, "
@@ -1328,13 +1331,14 @@ class ParallelConfig:
1331
from vllm.executor.executor_base import ExecutorBase
1332
from vllm.platforms import current_platform
1333
if self.distributed_executor_backend not in (
1331
- "ray", "mp", None) and not (isinstance(
1334
+ "ray", "mp", "uni", None) and not (isinstance(
1335
self.distributed_executor_backend, type) and issubclass(
1336
self.distributed_executor_backend, ExecutorBase)):
1337
raise ValueError(
1338
"Unrecognized distributed executor backend "
1339
f"{self.distributed_executor_backend}. Supported "
1337
- "values are 'ray', 'mp' or custom ExecutorBase subclass.")
1340
+ "values are 'ray', 'mp' 'uni', or custom ExecutorBase"
1341
+ " subclass.")
1342
if self.use_ray:
1343
from vllm.executor import ray_utils
1344
ray_utils.assert_ray_available()
vllm/distributed/parallel_state.py
+4
-2
@@ -862,12 +862,14 @@ def init_model_parallel_group(
862
) -> GroupCoordinator:
863
if use_custom_allreduce is None:
864
use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE
865
+ from vllm.platforms import current_platform
866
return GroupCoordinator(
867
group_ranks=group_ranks,
868
local_rank=local_rank,
869
torch_distributed_backend=backend,
869
- use_pynccl=True,
870
- use_custom_allreduce=use_custom_allreduce,
870
+ use_pynccl=current_platform.is_cuda_alike(),
871
+ use_custom_allreduce=current_platform.is_cuda_alike()
872
+ and use_custom_allreduce,
873
use_tpu_communicator=True,
874
use_hpu_communicator=True,
875
use_xpu_communicator=True,
vllm/engine/async_llm_engine.py
+6
-82
@@ -18,9 +18,7 @@ from vllm.engine.async_timeout import asyncio_timeout
18
from vllm.engine.llm_engine import LLMEngine, SchedulerOutputState
19
from vllm.engine.metrics_types import StatLoggerBase
20
from vllm.engine.protocol import EngineClient
21
-from vllm.executor.executor_base import ExecutorAsyncBase
22
-from vllm.executor.gpu_executor import GPUExecutorAsync
23
-from vllm.executor.ray_utils import initialize_ray_cluster
21
+from vllm.executor.executor_base import ExecutorBase
22
from vllm.inputs import PromptType
23
from vllm.inputs.preprocess import InputPreprocessor
24
from vllm.logger import init_logger
@@ -620,69 +618,9 @@ class AsyncLLMEngine(EngineClient):
618
rt.new_requests_event.set()
619
620
@classmethod
623
- def _get_executor_cls(
624
- cls, engine_config: VllmConfig) -> Type[ExecutorAsyncBase]:
625
- distributed_executor_backend = (
626
- engine_config.parallel_config.distributed_executor_backend)
627
- if isinstance(distributed_executor_backend, type):
628
- if not issubclass(distributed_executor_backend, ExecutorAsyncBase):
629
- raise TypeError(
630
- "distributed_executor_backend must be a subclass of "
631
- f"ExecutorAsyncBase. Got {distributed_executor_backend}.")
632
- executor_class = distributed_executor_backend
633
- elif engine_config.device_config.device_type == "neuron":
634
- from vllm.executor.neuron_executor import NeuronExecutorAsync
635
- executor_class = NeuronExecutorAsync
636
- elif engine_config.device_config.device_type == "tpu":
637
- if distributed_executor_backend == "ray":
638
- from vllm.executor.ray_tpu_executor import RayTPUExecutorAsync
639
- executor_class = RayTPUExecutorAsync
640
- else:
641
- assert distributed_executor_backend is None
642
- from vllm.executor.tpu_executor import TPUExecutorAsync
643
- executor_class = TPUExecutorAsync
644
- elif engine_config.device_config.device_type == "cpu":
645
- from vllm.executor.cpu_executor import CPUExecutorAsync
646
- executor_class = CPUExecutorAsync
647
- elif engine_config.device_config.device_type == "hpu":
648
- if distributed_executor_backend == "ray":
649
- initialize_ray_cluster(engine_config.parallel_config)
650
- from vllm.executor.ray_hpu_executor import RayHPUExecutorAsync
651
- executor_class = RayHPUExecutorAsync
652
- else:
653
- from vllm.executor.hpu_executor import HPUExecutorAsync
654
- executor_class = HPUExecutorAsync
655
- elif engine_config.device_config.device_type == "openvino":
656
- assert distributed_executor_backend is None, (
657
- "Distributed execution is not supported with "
658
- "the OpenVINO backend.")
659
- from vllm.executor.openvino_executor import OpenVINOExecutorAsync
660
- executor_class = OpenVINOExecutorAsync
661
- elif engine_config.device_config.device_type == "xpu":
662
- if distributed_executor_backend is None:
663
- from vllm.executor.xpu_executor import XPUExecutorAsync
664
- executor_class = XPUExecutorAsync
665
- elif distributed_executor_backend == "ray":
666
- from vllm.executor.ray_xpu_executor import RayXPUExecutorAsync
667
- executor_class = RayXPUExecutorAsync
668
- elif distributed_executor_backend == "mp":
669
- from vllm.executor.multiproc_xpu_executor import (
670
- MultiprocessingXPUExecutorAsync)
671
- executor_class = MultiprocessingXPUExecutorAsync
672
- else:
673
- raise RuntimeError(
674
- "Not supported distributed execution model on XPU device.")
675
- elif distributed_executor_backend == "ray":
676
- from vllm.executor.ray_gpu_executor import RayGPUExecutorAsync
677
- executor_class = RayGPUExecutorAsync
678
- elif distributed_executor_backend == "mp":
679
- from vllm.executor.multiproc_gpu_executor import (
680
- MultiprocessingGPUExecutorAsync)
681
- executor_class = MultiprocessingGPUExecutorAsync
682
- else:
683
- from vllm.executor.gpu_executor import GPUExecutorAsync
684
- executor_class = GPUExecutorAsync
685
- return executor_class
621
+ def _get_executor_cls(cls,
622
+ engine_config: VllmConfig) -> Type[ExecutorBase]:
623
+ return LLMEngine._get_executor_cls(engine_config)
624
625
@classmethod
626
def from_engine_args(
@@ -700,9 +638,6 @@ class AsyncLLMEngine(EngineClient):
638
639
executor_class = cls._get_executor_cls(engine_config)
640
703
- if executor_class.uses_ray:
704
- initialize_ray_cluster(engine_config.parallel_config)
705
-
641
# Create the async LLM engine.
642
engine = cls(
643
vllm_config=engine_config,
@@ -1242,23 +1177,12 @@ class AsyncLLMEngine(EngineClient):
1177
self.engine.remove_logger(logger_name=logger_name)
1178
1179
async def start_profile(self) -> None:
1245
- # using type instead of isinstance to check to avoid capturing
1246
- # inherited classes
1247
- if type(self.engine.model_executor) == GPUExecutorAsync: # noqa: E721
1248
- self.engine.model_executor.start_profile()
1249
- else:
1250
- self.engine.model_executor._run_workers("start_profile")
1180
+ self.engine.start_profile()
1181
1182
async def stop_profile(self) -> None:
1253
- # using type instead of isinstance to check to avoid capturing
1254
- # inherited classes
1255
- if type(self.engine.model_executor) == GPUExecutorAsync: # noqa: E721
1256
- self.engine.model_executor.stop_profile()
1257
- else:
1258
- self.engine.model_executor._run_workers("stop_profile")
1183
+ self.engine.stop_profile()
1184
1185
async def add_lora(self, lora_request: LoRARequest) -> None:
1261
- """Load a new LoRA adapter into the engine for future requests."""
1186
self.engine.add_lora(lora_request)
1187
1188
vllm/engine/llm_engine.py
+22
-72
@@ -28,8 +28,6 @@ from vllm.engine.output_processor.util import create_output_by_sequence_group
28
from vllm.entrypoints.openai.logits_processors import (
29
get_logits_processors as get_openai_logits_processors)
30
from vllm.executor.executor_base import ExecutorBase
31
-from vllm.executor.gpu_executor import GPUExecutor
32
-from vllm.executor.ray_utils import initialize_ray_cluster
31
from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
32
PromptType, SingletonInputsAdapter)
33
from vllm.inputs.parse import is_encoder_decoder_inputs, is_token_prompt
@@ -442,64 +440,26 @@ class LLMEngine:
440
raise TypeError(
441
"distributed_executor_backend must be a subclass of "
442
f"ExecutorBase. Got {distributed_executor_backend}.")
445
- if distributed_executor_backend.uses_ray: # type: ignore
446
- initialize_ray_cluster(engine_config.parallel_config)
443
executor_class = distributed_executor_backend
448
- elif engine_config.device_config.device_type == "neuron":
449
- from vllm.executor.neuron_executor import NeuronExecutor
450
- executor_class = NeuronExecutor
451
- elif engine_config.device_config.device_type == "tpu":
444
+ elif engine_config.parallel_config.world_size > 1:
445
if distributed_executor_backend == "ray":
453
- initialize_ray_cluster(engine_config.parallel_config)
454
- from vllm.executor.ray_tpu_executor import RayTPUExecutor
455
- executor_class = RayTPUExecutor
456
- else:
457
- assert distributed_executor_backend is None
458
- from vllm.executor.tpu_executor import TPUExecutor
459
- executor_class = TPUExecutor
460
- elif engine_config.device_config.device_type == "cpu":
461
- from vllm.executor.cpu_executor import CPUExecutor
462
- executor_class = CPUExecutor
463
- elif engine_config.device_config.device_type == "hpu":
464
- if distributed_executor_backend == "ray":
465
- initialize_ray_cluster(engine_config.parallel_config)
466
- from vllm.executor.ray_hpu_executor import RayHPUExecutor
467
- executor_class = RayHPUExecutor
468
- else:
469
- from vllm.executor.hpu_executor import HPUExecutor
470
- executor_class = HPUExecutor
471
- elif engine_config.device_config.device_type == "openvino":
472
- from vllm.executor.openvino_executor import OpenVINOExecutor
473
- executor_class = OpenVINOExecutor
474
- elif engine_config.device_config.device_type == "xpu":
475
- if distributed_executor_backend == "ray":
476
- initialize_ray_cluster(engine_config.parallel_config)
477
- from vllm.executor.ray_xpu_executor import RayXPUExecutor
478
- executor_class = RayXPUExecutor
446
+ from vllm.executor.ray_distributed_executor import (
447
+ RayDistributedExecutor)
448
+ executor_class = RayDistributedExecutor
449
elif distributed_executor_backend == "mp":
480
- # FIXME(kunshang):
481
- # spawn needs calling `if __name__ == '__main__':``
482
- # fork is not supported for xpu start new process.
483
- logger.error(
484
- "Both start methods (spawn and fork) have issue "
485
- "on XPU if you use mp backend, Please try ray instead.")
486
- else:
487
- from vllm.executor.xpu_executor import XPUExecutor
488
- executor_class = XPUExecutor
489
- elif distributed_executor_backend == "ray":
490
- initialize_ray_cluster(engine_config.parallel_config)
491
- from vllm.executor.ray_gpu_executor import RayGPUExecutor
492
- executor_class = RayGPUExecutor
493
- elif distributed_executor_backend == "mp":
494
- from vllm.executor.multiproc_gpu_executor import (
495
- MultiprocessingGPUExecutor)
496
- assert not envs.VLLM_USE_RAY_SPMD_WORKER, (
497
- "multiprocessing distributed executor backend does not "
498
- "support VLLM_USE_RAY_SPMD_WORKER=1")
499
- executor_class = MultiprocessingGPUExecutor
450
+ from vllm.executor.mp_distributed_executor import (
451
+ MultiprocessingDistributedExecutor)
452
+ assert not envs.VLLM_USE_RAY_SPMD_WORKER, (
453
+ "multiprocessing distributed executor backend does not "
454
+ "support VLLM_USE_RAY_SPMD_WORKER=1")
455
+ executor_class = MultiprocessingDistributedExecutor
456
+ elif distributed_executor_backend == "uni":
457
+ # JAX-style, single-process, multi-device executor.
458
+ from vllm.executor.uniproc_executor import UniProcExecutor
459
+ executor_class = UniProcExecutor
460
else:
501
- from vllm.executor.gpu_executor import GPUExecutor
502
- executor_class = GPUExecutor
461
+ from vllm.executor.uniproc_executor import UniProcExecutor
462
+ executor_class = UniProcExecutor
463
return executor_class
464
465
@classmethod
@@ -1845,27 +1805,17 @@ class LLMEngine:
1805
def list_prompt_adapters(self) -> List[int]:
1806
return self.model_executor.list_prompt_adapters()
1807
1808
+ def start_profile(self) -> None:
1809
+ self.model_executor.start_profile()
1810
+
1811
+ def stop_profile(self) -> None:
1812
+ self.model_executor.stop_profile()
1813
+
1814
def check_health(self) -> None:
1815
if self.tokenizer:
1816
self.tokenizer.check_health()
1817
self.model_executor.check_health()
1818
1853
- def start_profile(self) -> None:
1854
- # using type instead of isinstance to check to avoid capturing
1855
- # inherited classes (MultiprocessingGPUExecutor)
1856
- if type(self.model_executor) == GPUExecutor: # noqa: E721
1857
- self.model_executor.start_profile()
1858
- else:
1859
- self.model_executor._run_workers("start_profile")
1860
-
1861
- def stop_profile(self) -> None:
1862
- # using type instead of isinstance to check to avoid capturing
1863
- # inherited classes (MultiprocessingGPUExecutor)
1864
- if type(self.model_executor) == GPUExecutor: # noqa: E721
1865
- self.model_executor.stop_profile()
1866
- else:
1867
- self.model_executor._run_workers("stop_profile")
1868
-
1819
def is_tracing_enabled(self) -> bool:
1820
return self.tracer is not None
1821
vllm/engine/multiprocessing/engine.py
+2
-9
@@ -20,7 +20,6 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
20
RPCStartupResponse,
21
RPCUProfileRequest)
22
# yapf: enable
23
-from vllm.executor.gpu_executor import GPUExecutor
23
from vllm.logger import init_logger
24
from vllm.outputs import RequestOutput
25
from vllm.usage.usage_lib import UsageContext
@@ -356,16 +355,10 @@ class MQLLMEngine:
355
self._errored_with = e
356
357
def start_profile(self) -> None:
359
- if type(self.engine.model_executor) is GPUExecutor:
360
- self.engine.model_executor.start_profile()
361
- else:
362
- self.engine.model_executor._run_workers("start_profile")
358
+ self.engine.start_profile()
359
360
def stop_profile(self) -> None:
365
- if type(self.engine.model_executor) is GPUExecutor:
366
- self.engine.model_executor.stop_profile()
367
- else:
368
- self.engine.model_executor._run_workers("stop_profile")
361
+ self.engine.stop_profile()
362
363
364
def signal_handler(*_) -> None:
vllm/executor/cpu_executor.py
deleted
-299
@@ -1,299 +0,0 @@
1
-import os
2
-from functools import partial
3
-from typing import Any, Awaitable, List, Optional, Set, Tuple, Union
4
-
5
-from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
6
-from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
7
- ResultHandler, WorkerMonitor)
8
-from vllm.logger import init_logger
9
-from vllm.lora.request import LoRARequest
10
-from vllm.model_executor.layers.sampler import SamplerOutput
11
-from vllm.prompt_adapter.request import PromptAdapterRequest
12
-from vllm.sequence import ExecuteModelRequest
13
-from vllm.utils import get_distributed_init_method, get_open_port, make_async
14
-from vllm.worker.worker_base import WorkerWrapperBase
15
-
16
-logger = init_logger(__name__)
17
-
18
-
19
-class CPUExecutor(ExecutorBase):
20
-
21
- uses_ray: bool = False
22
-
23
- def _init_executor(self) -> None:
24
- assert self.device_config.device_type == "cpu"
25
-
26
- #
27
- # Environment variables for CPU executor
28
- #
29
-
30
- # Disable torch async compiling which won't work with daemonic processes
31
- os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
32
-
33
- # Intel OpenMP setting
34
- ld_prealod_str = os.getenv("LD_PRELOAD", "")
35
- if "libiomp5.so" in ld_prealod_str:
36
- # The time(milliseconds) that a thread should wait after
37
- # completing the execution of a parallel region, before sleeping.
38
- os.environ['KMP_BLOCKTIME'] = "1"
39
- # Prevents the CPU to run into low performance state
40
- os.environ['KMP_TPAUSE'] = "0"
41
- # Provides fine granularity parallelism
42
- os.environ['KMP_FORKJOIN_BARRIER_PATTERN'] = "dist,dist"
43
- os.environ['KMP_PLAIN_BARRIER_PATTERN'] = "dist,dist"
44
- os.environ['KMP_REDUCTION_BARRIER_PATTERN'] = "dist,dist"
45
-
46
- # To hint IPEX uses shared memory based AllReduce
47
- os.environ["LOCAL_WORLD_SIZE"] = str(
48
- self.parallel_config.tensor_parallel_size)
49
-
50
- # Multiprocessing-based executor does not support multi-node setting.
51
- # Since it only works for single node, we can use the loopback address
52
- # 127.0.0.1 for communication.
53
- ip = "127.0.0.1"
54
- port = get_open_port()
55
- self.distributed_init_method = get_distributed_init_method(ip, port)
56
-
57
- is_async = isinstance(self, CPUExecutorAsync)
58
-
59
- world_size = self.parallel_config.tensor_parallel_size
60
- result_handler = ResultHandler()
61
- self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None
62
- self.workers = []
63
-
64
- if is_async:
65
- self.workers = [
66
- ProcessWorkerWrapper(
67
- result_handler,
68
- partial(
69
- self._create_worker,
70
- rank=rank,
71
- local_rank=rank,
72
- )) for rank in range(0, world_size)
73
- ]
74
- self.driver_worker = self.workers[0]
75
- self.workers = self.workers[1:]
76
- self.driver_method_invoker = _async_driver_method_invoker
77
- else:
78
- self.driver_worker = self._create_worker()
79
- self.driver_method_invoker = _driver_method_invoker
80
-
81
- if world_size != 1:
82
- self.workers = [
83
- ProcessWorkerWrapper(
84
- result_handler,
85
- partial(
86
- self._create_worker,
87
- rank=rank,
88
- local_rank=rank,
89
- )) for rank in range(1, world_size)
90
- ]
91
-
92
- self.worker_monitor = None
93
- if world_size != 1 or is_async:
94
- if is_async:
95
- async_worker_list = self.workers + [self.driver_worker]
96
- else:
97
- async_worker_list = self.workers
98
- self.worker_monitor = WorkerMonitor(async_worker_list,
99
- result_handler)
100
- result_handler.start()
101
- self.worker_monitor.start()
102
-
103
- self._run_workers("init_device")
104
- self._run_workers("load_model")
105
-
106
- def _create_worker(
107
- self,
108
- local_rank: int = 0,
109
- rank: int = 0,
110
- ):
111
-
112
- wrapper = WorkerWrapperBase(vllm_config=self.vllm_config)
113
-
114
- assert self.distributed_init_method is not None
115
-
116
- kwargs = dict(
117
- vllm_config=self.vllm_config,
118
- local_rank=local_rank,
119
- rank=rank,
120
- distributed_init_method=self.distributed_init_method,
121
- kv_cache_dtype=self.cache_config.cache_dtype,
122
- is_driver_worker=rank == 0,
123
- )
124
- wrapper.init_worker(**kwargs)
125
-
126
- return wrapper.worker
127
-
128
- def _run_workers(
129
- self,
130
- method: str,
131
- *args,
132
- async_run_remote_workers_only: bool = False,
133
- max_concurrent_workers: Optional[int] = None,
134
- **kwargs,
135
- ) -> Any:
136
- """Runs the given method on all workers.
137
-
138
- Args:
139
- async_run_remote_workers_only: If True the method will be run only
140
- in the remote workers, not the driver worker. It will also be
141
- run asynchronously and return a list of futures rather than
142
- blocking on the results.
143
- """
144
-
145
- if max_concurrent_workers:
146
- raise NotImplementedError(
147
- "max_concurrent_workers is not supported yet.")
148
-
149
- # Start the workers first.
150
- worker_outputs = [
151
- worker.execute_method(method, *args, **kwargs)
152
- for worker in self.workers
153
- ]
154
-
155
- if async_run_remote_workers_only:
156
- # Just return futures
157
- return worker_outputs
158
-
159
- driver_worker_output = self.driver_method_invoker(
160
- self.driver_worker, method, *args, **kwargs)
161
-
162
- # Get the results of the workers.
163
- return [driver_worker_output
164
- ] + [output.get() for output in worker_outputs]
165
-
166
- def determine_num_available_blocks(self) -> Tuple[int, int]:
167
- """Determine the number of available KV blocks by invoking the
168
- underlying worker.
169
- """
170
- return self.driver_method_invoker(self.driver_worker,
171
- "determine_num_available_blocks")
172
-
173
- def initialize_cache(self, num_gpu_blocks: int,
174
- num_cpu_blocks: int) -> None:
175
- """Initialize the KV cache by invoking the underlying worker.
176
- """
177
- # NOTE: We log here to avoid multiple logs when number of workers is
178
- # greater than one. We could log in the engine, but not all executors
179
- # have GPUs.
180
- # NOTE: `cpu block` for CPU backend is located on CPU memory but is
181
- # referred as `gpu block`. Because we want to reuse the existing block
182
- # management procedure.
183
- logger.info("# CPU blocks: %d", num_gpu_blocks)
184
-
185
- self._run_workers("initialize_cache",
186
- num_gpu_blocks=num_gpu_blocks,
187
- num_cpu_blocks=num_cpu_blocks)
188
-
189
- def execute_model(
190
- self,
191
- execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
192
- if (self.parallel_config.tensor_parallel_size > 1
193
- and self.parallel_worker_tasks is None):
194
- self.parallel_worker_tasks = self._run_workers(
195
- "start_worker_execution_loop",
196
- async_run_remote_workers_only=True,
197
- )
198
- output = self.driver_method_invoker(self.driver_worker,
199
- "execute_model", execute_model_req)
200
- return output
201
-
202
- def stop_remote_worker_execution_loop(self) -> None:
203
- if self.parallel_worker_tasks is None:
204
- return
205
- """
206
- Passing None will cause the driver to stop the model execution
207
- loop running in each of the remote workers.
208
- """
209
- self.driver_method_invoker(self.driver_worker, "execute_model", None)
210
- parallel_worker_tasks = self.parallel_worker_tasks
211
- self.parallel_worker_tasks = None
212
- # Ensure that workers exit model loop cleanly
213
- # (this will raise otherwise)
214
- self._wait_for_tasks_completion(parallel_worker_tasks)
215
-
216
- def add_lora(self, lora_request: LoRARequest) -> bool:
217
- return all(self._run_workers("add_lora", lora_request))
218
-
219
- def remove_lora(self, lora_id: int) -> bool:
220
- return all(self._run_workers("remove_lora", lora_id))
221
-
222
- def pin_lora(self, lora_id: int) -> bool:
223
- assert lora_id > 0, "lora_id must be greater than 0."
224
- return all(self._run_workers(
225
- "pin_lora",
226
- lora_id=lora_id,
227
- ))
228
-
229
- def list_loras(self) -> Set[int]:
230
- return self.driver_method_invoker(self.driver_worker, "list_loras")
231
-
232
- def add_prompt_adapter(
233
- self, prompt_adapter_request: PromptAdapterRequest) -> bool:
234
- return all(
235
- self._run_workers(
236
- "add_prompt_adapter",
237
- prompt_adapter_request,
238
- ))
239
-
240
- def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
241
- return all(
242
- self._run_workers(
243
- "remove_prompt_adapter",
244
- prompt_adapter_id,
245
- ))
246
-
247
- def list_prompt_adapters(self) -> Set[int]:
248
- return self.driver_method_invoker(self.driver_worker,
249
- "list_prompt_adapters")
250
-
251
- def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
252
- return all(self._run_workers(
253
- "pin_prompt_adapter",
254
- prompt_adapter_id,
255
- ))
256
-
257
- def check_health(self) -> None:
258
- """Raises an error if engine is unhealthy."""
259
- if self.worker_monitor is not None and not self.worker_monitor.is_alive(
260
- ):
261
- raise RuntimeError("Worker processes are not running")
262
-
263
- def shutdown(self):
264
- if (worker_monitor := getattr(self, "worker_monitor",
265
- None)) is not None:
266
- worker_monitor.close()
267
-
268
- def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
269
- """Wait for futures returned from _run_workers() with
270
- async_run_remote_workers_only to complete."""
271
- for result in parallel_worker_tasks:
272
- result.get()
273
-
274
- def start_profile(self) -> None:
275
- self.driver_method_invoker(self.driver_worker, "start_profile")
276
-
277
- def stop_profile(self) -> None:
278
- self.driver_method_invoker(self.driver_worker, "stop_profile")
279
-
280
-
281
-class CPUExecutorAsync(CPUExecutor, ExecutorAsyncBase):
282
-
283
- async def execute_model_async(
284
- self,
285
- execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
286
- output = await make_async(self.execute_model
287
- )(execute_model_req=execute_model_req, )
288
- return output
289
-
290
- async def check_health_async(self) -> None:
291
- self.check_health()
292
-
293
-
294
-def _driver_method_invoker(driver, method: str, *args, **kwargs):
295
- return getattr(driver, method)(*args, **kwargs)
296
-
297
-
298
-def _async_driver_method_invoker(driver, method: str, *args, **kwargs):
299
- return driver.execute_method(method, *args, **kwargs).get()
vllm/executor/distributed_gpu_executor.py
deleted
-212
@@ -1,212 +0,0 @@
1
-import asyncio
2
-from abc import abstractmethod
3
-from typing import Any, Awaitable, Dict, List, Optional, Set, Tuple, Union
4
-
5
-from vllm.executor.executor_base import ExecutorAsyncBase
6
-from vllm.executor.gpu_executor import GPUExecutor
7
-from vllm.logger import init_logger
8
-from vllm.lora.request import LoRARequest
9
-from vllm.model_executor.layers.sampler import SamplerOutput
10
-from vllm.sequence import ExecuteModelRequest
11
-
12
-logger = init_logger(__name__)
13
-
14
-
15
-class DistributedGPUExecutor(GPUExecutor):
16
- """Abstract superclass of multi-GPU executor implementations."""
17
-
18
- def __init__(self, *args, **kwargs):
19
- # This is non-None when the execute model loop is running
20
- # in the parallel workers. It's a coroutine in the AsyncLLMEngine case.
21
- self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None
22
- # Updated by implementations that require additional args to be passed
23
- # to the _run_workers execute_model call
24
- self.extra_execute_model_run_workers_kwargs: Dict[str, Any] = {}
25
-
26
- super().__init__(*args, **kwargs)
27
-
28
- def determine_num_available_blocks(self) -> Tuple[int, int]:
29
- """Determine the number of available KV blocks.
30
-
31
- This invokes `determine_num_available_blocks` on each worker and takes
32
- the min of the results, guaranteeing that the selected cache sizes are
33
- compatible with all workers.
34
-
35
- Returns:
36
- - tuple[num_gpu_blocks, num_cpu_blocks]
37
- """
38
- # Get the maximum number of blocks that can be allocated on GPU and CPU.
39
- num_blocks = self._run_workers("determine_num_available_blocks", )
40
-
41
- # Since we use a shared centralized controller, we take the minimum
42
- # number of blocks across all workers to make sure all the memory
43
- # operators can be applied to all workers.
44
- num_gpu_blocks = min(b[0] for b in num_blocks)
45
- num_cpu_blocks = min(b[1] for b in num_blocks)
46
-
47
- return num_gpu_blocks, num_cpu_blocks
48
-
49
- def initialize_cache(self, num_gpu_blocks: int,
50
- num_cpu_blocks: int) -> None:
51
- """Initialize the KV cache in all workers.
52
- """
53
-
54
- # NOTE: We log here to avoid multiple logs when number of workers is
55
- # greater than one. We could log in the engine, but not all executors
56
- # have GPUs.
57
- logger.info("# GPU blocks: %d, # CPU blocks: %d", num_gpu_blocks,
58
- num_cpu_blocks)
59
- max_concurrency = (num_gpu_blocks * self.cache_config.block_size /
60
- self.model_config.max_model_len)
61
- logger.info("Maximum concurrency for %s tokens per request: %.2fx",
62
- self.model_config.max_model_len, max_concurrency)
63
-
64
- self.cache_config.num_gpu_blocks = num_gpu_blocks
65
- self.cache_config.num_cpu_blocks = num_cpu_blocks
66
-
67
- self._run_workers("initialize_cache",
68
- num_gpu_blocks=num_gpu_blocks,
69
- num_cpu_blocks=num_cpu_blocks)
70
-
71
- def execute_model(
72
- self,
73
- execute_model_req: ExecuteModelRequest,
74
- ) -> List[SamplerOutput]:
75
- if self.parallel_worker_tasks is None:
76
- self.parallel_worker_tasks = self._run_workers(
77
- "start_worker_execution_loop",
78
- async_run_tensor_parallel_workers_only=True,
79
- **self.extra_execute_model_run_workers_kwargs)
80
-
81
- # Only the driver worker returns the sampling results.
82
- driver_outputs = self._driver_execute_model(execute_model_req)
83
- assert driver_outputs is not None
84
- return driver_outputs
85
-
86
- def stop_remote_worker_execution_loop(self) -> None:
87
- if self.parallel_worker_tasks is None:
88
- return
89
-
90
- self._driver_execute_model(execute_model_req=None)
91
- parallel_worker_tasks = self.parallel_worker_tasks
92
- self.parallel_worker_tasks = None
93
- # Ensure that workers exit model loop cleanly
94
- # (this will raise otherwise)
95
- self._wait_for_tasks_completion(parallel_worker_tasks)
96
-
97
- def add_lora(self, lora_request: LoRARequest) -> bool:
98
- assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
99
- return self._run_workers(
100
- "add_lora",
101
- lora_request=lora_request,
102
- )
103
-
104
- def remove_lora(self, lora_id: int) -> bool:
105
- assert lora_id > 0, "lora_id must be greater than 0."
106
- return self._run_workers(
107
- "remove_lora",
108
- lora_id=lora_id,
109
- )
110
-
111
- def pin_lora(self, lora_id: int) -> bool:
112
- assert lora_id > 0, "lora_id must be greater than 0."
113
- return self._run_workers(
114
- "pin_lora",
115
- lora_id=lora_id,
116
- )
117
-
118
- def list_loras(self) -> Set[int]:
119
- return self._run_workers("list_loras")
120
-
121
- def save_sharded_state(
122
- self,
123
- path: str,
124
- pattern: Optional[str] = None,
125
- max_size: Optional[int] = None,
126
- ) -> None:
127
- self._run_workers("save_sharded_state",
128
- path=path,
129
- pattern=pattern,
130
- max_size=max_size)
131
-
132
- @abstractmethod
133
- def _driver_execute_model(
134
- self, execute_model_req: Optional[ExecuteModelRequest]
135
- ) -> Optional[List[SamplerOutput]]:
136
- """Run execute_model in the driver worker.
137
-
138
- Passing None will cause the driver to stop the model execution loop
139
- running in each of the remote workers. In this case, this method
140
- returns None. Otherwise, this method returns the model output.
141
- """
142
- raise NotImplementedError
143
-
144
- @abstractmethod
145
- def _run_workers(
146
- self,
147
- method: str,
148
- *args,
149
- async_run_tensor_parallel_workers_only: bool = False,
150
- max_concurrent_workers: Optional[int] = None,
151
- **kwargs,
152
- ) -> Any:
153
- """Runs the given method on all workers.
154
-
155
- Args:
156
- async_run_tensor_parallel_workers_only: If True the method will be
157
- run only in the remote TP workers, not the driver worker.
158
- It will also be run asynchronously and return a list of futures
159
- rather than blocking on the results.
160
- """
161
- raise NotImplementedError
162
-
163
- @abstractmethod
164
- def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
165
- """Wait for futures returned from _run_workers() with
166
- async_run_remote_workers_only to complete."""
167
- raise NotImplementedError
168
-
169
-
170
-class DistributedGPUExecutorAsync(DistributedGPUExecutor, ExecutorAsyncBase):
171
-
172
- async def execute_model_async(
173
- self,
174
- execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
175
- if self.parallel_worker_tasks is None:
176
- # Start model execution loop running in the parallel workers
177
- self.parallel_worker_tasks = asyncio.create_task(
178
- self._start_worker_execution_loop())
179
-
180
- # Only the driver worker returns the sampling results.
181
- return await self._driver_execute_model_async(execute_model_req)
182
-
183
- async def stop_remote_worker_execution_loop_async(self) -> None:
184
- if self.parallel_worker_tasks is None:
185
- return
186
-
187
- await self._driver_execute_model_async()
188
- parallel_worker_tasks = self.parallel_worker_tasks
189
- self.parallel_worker_tasks = None
190
- # Ensure that workers exit model loop cleanly
191
- # (this will raise otherwise)
192
- await parallel_worker_tasks
193
-
194
- @abstractmethod
195
- async def _driver_execute_model_async(
196
- self,
197
- execute_model_req: Optional[ExecuteModelRequest] = None,
198
- ) -> List[SamplerOutput]:
199
- """Execute the model asynchronously in the driver worker.
200
-
201
- Passing None will cause the driver to stop the model execution
202
- loop running in each of the remote workers.
203
- """
204
- raise NotImplementedError
205
-
206
- @abstractmethod
207
- async def _start_worker_execution_loop(self):
208
- """Run execution loop on all workers. It guarantees all workers run
209
- the loop or None of them is running the loop. Loop can be stopped by
210
- `stop_remote_worker_execution_loop`.
211
- The API is idempotent (guarantee only 1 loop run at any moment)."""
212
- raise NotImplementedError
vllm/executor/executor_base.py
+232
-35
@@ -1,18 +1,24 @@
1
+import asyncio
2
from abc import ABC, abstractmethod
2
-from typing import List, Optional, Set, Tuple
3
+from typing import Any, Awaitable, Dict, List, Optional, Set, Tuple, Union
4
5
from vllm.config import VllmConfig
6
+from vllm.logger import init_logger
7
from vllm.lora.request import LoRARequest
8
from vllm.model_executor.layers.sampler import SamplerOutput
9
+from vllm.platforms import current_platform
10
from vllm.prompt_adapter.request import PromptAdapterRequest
8
-from vllm.sequence import ExecuteModelRequest
11
+from vllm.sequence import ExecuteModelRequest, PoolerOutput
12
+from vllm.utils import make_async
13
+
14
+logger = init_logger(__name__)
15
16
17
class ExecutorBase(ABC):
18
"""Base class for all executors.
19
14
- An executor is responsible for executing the model on a specific device
15
- type (e.g., CPU, GPU, Neuron, etc.). Or it can be a distributed executor
20
+ An executor is responsible for executing the model on one device,
21
+ or it can be a distributed executor
22
that can execute the model on multiple devices.
23
"""
24
@@ -40,6 +46,20 @@ class ExecutorBase(ABC):
46
pass
47
48
@abstractmethod
49
+ def collective_rpc(self,
50
+ method: str,
51
+ timeout: Optional[float] = None,
52
+ args: Tuple = (),
53
+ kwargs: Optional[Dict] = None) -> List[Any]:
54
+ """
55
+ The main interface of the executor to run a method on all workers,
56
+ with homogeneous arguments.
57
+ If the args are heterogeneous, then we can pack them into a list,
58
+ and unpack them in the method of every worker, because every worker
59
+ knows their own rank.
60
+ """
61
+ pass
62
+
63
def determine_num_available_blocks(self) -> Tuple[int, int]:
64
"""Determine the number of available blocks for the GPU KV cache and
65
swappable CPU KV cache.
@@ -53,58 +73,113 @@ class ExecutorBase(ABC):
73
num_cpu_blocks refers to "swapped" blocks in CPU memory and cannot be
74
appended to.
75
"""
56
- raise NotImplementedError
76
+ results = self.collective_rpc("determine_num_available_blocks")
77
+ a = min([r[0] for r in results])
78
+ b = min([r[1] for r in results])
79
+ return a, b
80
58
- @abstractmethod
59
- def initialize_cache(self, num_gpu_blocks: int,
60
- num_cpu_blocks: int) -> None:
61
- """Initialize the KV cache with the given size in blocks.
81
+ def initialize(self, num_gpu_blocks: int) -> None:
82
"""
63
- raise NotImplementedError
83
+ Initialize the KV caches and begin the model execution loop of the
84
+ underlying workers.
85
+ For V1 compatibility.
86
+ """
87
+ logger.info("# GPU blocks: %d", num_gpu_blocks)
88
+ self.collective_rpc("initialize_cache", args=(num_gpu_blocks, ))
89
+ self.collective_rpc("compile_or_warm_up_model")
90
+
91
+ def initialize_cache(self, num_gpu_blocks: int, num_cpu_blocks) -> None:
92
+ """Initialize the KV cache by invoking the underlying worker.
93
+ """
94
+ # NOTE: This is logged in the executor because there can be >1 workers.
95
+ logger.info("# %s blocks: %d, # CPU blocks: %d",
96
+ current_platform.dispatch_key, num_gpu_blocks,
97
+ num_cpu_blocks)
98
+ max_concurrency = (num_gpu_blocks * self.cache_config.block_size /
99
+ self.model_config.max_model_len)
100
+ logger.info("Maximum concurrency for %s tokens per request: %.2fx",
101
+ self.model_config.max_model_len, max_concurrency)
102
+
103
+ self.cache_config.num_gpu_blocks = num_gpu_blocks
104
+ self.cache_config.num_cpu_blocks = num_cpu_blocks
105
+
106
+ self.collective_rpc("initialize_cache",
107
+ args=(num_gpu_blocks, num_cpu_blocks))
108
65
- @abstractmethod
109
def execute_model(
110
self, execute_model_req: ExecuteModelRequest
68
- ) -> Optional[List[SamplerOutput]]:
69
- """Executes at least one model step on the given sequences."""
70
- raise NotImplementedError
111
+ ) -> Optional[List[Union[SamplerOutput, PoolerOutput]]]:
112
+ output = self.collective_rpc("execute_model",
113
+ args=(execute_model_req, ))
114
+ return output[0]
115
116
def stop_remote_worker_execution_loop(self) -> None:
117
"""Releases parallel workers from model loop."""
118
return
119
76
- @abstractmethod
120
def add_lora(self, lora_request: LoRARequest) -> bool:
78
- raise NotImplementedError
121
+ assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
122
+ return all(self.collective_rpc("add_lora", args=(lora_request, )))
123
80
- @abstractmethod
124
def remove_lora(self, lora_id: int) -> bool:
82
- raise NotImplementedError
125
+ assert lora_id > 0, "lora_id must be greater than 0."
126
+ return all(self.collective_rpc("remove_lora", args=(lora_id, )))
127
84
- @abstractmethod
128
def pin_lora(self, lora_id: int) -> bool:
86
- raise NotImplementedError # type: ignore
129
+ assert lora_id > 0, "lora_id must be greater than 0."
130
+ return all(self.collective_rpc("pin_lora", args=(lora_id, )))
131
88
- @abstractmethod
132
def list_loras(self) -> Set[int]:
90
- raise NotImplementedError
133
+ sets = self.collective_rpc("list_loras")
134
+ for s in sets:
135
+ assert s == sets[0], "All workers should have the same LORAs."
136
+ return sets[0]
137
92
- @abstractmethod
138
def add_prompt_adapter(
139
self, prompt_adapter_request: PromptAdapterRequest) -> bool:
95
- raise NotImplementedError
140
+ assert prompt_adapter_request.prompt_adapter_id > 0, \
141
+ "prompt_adapter_id must be greater than 0."
142
+ return all(
143
+ self.collective_rpc("add_prompt_adapter",
144
+ args=(prompt_adapter_request, )))
145
97
- @abstractmethod
146
def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
99
- raise NotImplementedError
147
+ assert prompt_adapter_id > 0, \
148
+ "prompt_adapter_id must be greater than 0."
149
+ return all(
150
+ self.collective_rpc("remove_prompt_adapter",
151
+ args=(prompt_adapter_id, )))
152
101
- @abstractmethod
153
def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
103
- raise NotImplementedError # type: ignore
154
+ assert prompt_adapter_id > 0, \
155
+ "prompt_adapter_id must be greater than 0."
156
+ return all(
157
+ self.collective_rpc("pin_prompt_adapter",
158
+ args=(prompt_adapter_id, )))
159
105
- @abstractmethod
160
def list_prompt_adapters(self) -> Set[int]:
107
- raise NotImplementedError
161
+ sets = self.collective_rpc("list_prompt_adapters")
162
+ for s in sets:
163
+ assert (s == sets[0]
164
+ ), "All workers should have the same prompt adapters."
165
+ return sets[0]
166
+
167
+ def start_profile(self) -> None:
168
+ self.collective_rpc("start_profile")
169
+
170
+ def stop_profile(self) -> None:
171
+ self.collective_rpc("stop_profile")
172
+
173
+ def save_sharded_state(
174
+ self,
175
+ path: str,
176
+ pattern: Optional[str] = None,
177
+ max_size: Optional[int] = None,
178
+ ) -> None:
179
+ self.collective_rpc("save_sharded_state",
180
+ kwargs=dict(path=path,
181
+ pattern=pattern,
182
+ max_size=max_size))
183
184
@abstractmethod
185
def check_health(self) -> None:
@@ -119,15 +194,12 @@ class ExecutorBase(ABC):
194
def __del__(self):
195
self.shutdown()
196
122
-
123
-class ExecutorAsyncBase(ExecutorBase):
124
-
125
- @abstractmethod
197
async def execute_model_async(
198
self,
199
execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
200
"""Executes one model step on the given sequences."""
130
- raise NotImplementedError
201
+ output = await make_async(self.execute_model)(execute_model_req)
202
+ return output
203
204
async def stop_remote_worker_execution_loop_async(self) -> None:
205
"""Releases parallel workers from model loop."""
@@ -137,3 +209,128 @@ class ExecutorAsyncBase(ExecutorBase):
209
"""Checks if the executor is healthy. If not, it should raise an
210
exception."""
211
self.check_health()
212
+
213
+
214
+class DistributedExecutorBase(ExecutorBase):
215
+ """Abstract superclass of distributed executor implementations."""
216
+
217
+ def __init__(self, *args, **kwargs):
218
+ # This is non-None when the execute model loop is running
219
+ # in the parallel workers. It's a coroutine in the AsyncLLMEngine case.
220
+ self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None
221
+
222
+ super().__init__(*args, **kwargs)
223
+
224
+ def execute_model(
225
+ self,
226
+ execute_model_req: ExecuteModelRequest,
227
+ ) -> List[SamplerOutput]:
228
+ # TODO: unify into collective_rpc
229
+ if self.parallel_worker_tasks is None:
230
+ self.parallel_worker_tasks = self._run_workers(
231
+ "start_worker_execution_loop",
232
+ async_run_tensor_parallel_workers_only=True)
233
+
234
+ # Only the driver worker returns the sampling results.
235
+ driver_outputs = self._driver_execute_model(execute_model_req)
236
+ assert driver_outputs is not None
237
+ return driver_outputs
238
+
239
+ def stop_remote_worker_execution_loop(self) -> None:
240
+ if self.parallel_worker_tasks is None:
241
+ return
242
+
243
+ self._driver_execute_model(execute_model_req=None)
244
+ parallel_worker_tasks = self.parallel_worker_tasks
245
+ self.parallel_worker_tasks = None
246
+ # Ensure that workers exit model loop cleanly
247
+ # (this will raise otherwise)
248
+ self._wait_for_tasks_completion(parallel_worker_tasks)
249
+
250
+ @abstractmethod
251
+ def _driver_execute_model(
252
+ self, execute_model_req: Optional[ExecuteModelRequest]
253
+ ) -> Optional[List[SamplerOutput]]:
254
+ """Run execute_model in the driver worker.
255
+
256
+ Passing None will cause the driver to stop the model execution loop
257
+ running in each of the remote workers. In this case, this method
258
+ returns None. Otherwise, this method returns the model output.
259
+ """
260
+ raise NotImplementedError
261
+
262
+ def collective_rpc(self,
263
+ method: str,
264
+ timeout: Optional[float] = None,
265
+ args: Tuple = (),
266
+ kwargs: Optional[Dict] = None) -> List[Any]:
267
+ return self._run_workers(method, *args, **(kwargs or {}))
268
+
269
+ @abstractmethod
270
+ def _run_workers(
271
+ self,
272
+ method: str,
273
+ *args,
274
+ async_run_tensor_parallel_workers_only: bool = False,
275
+ max_concurrent_workers: Optional[int] = None,
276
+ **kwargs,
277
+ ) -> Any:
278
+ """Runs the given method on all workers.
279
+
280
+ Args:
281
+ async_run_tensor_parallel_workers_only: If True the method will be
282
+ run only in the remote TP workers, not the driver worker.
283
+ It will also be run asynchronously and return a list of futures
284
+ rather than blocking on the results.
285
+
286
+ # TODO: simplify and merge with collective_rpc
287
+ """
288
+ raise NotImplementedError
289
+
290
+ @abstractmethod
291
+ def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
292
+ """Wait for futures returned from _run_workers() with
293
+ async_run_remote_workers_only to complete."""
294
+ raise NotImplementedError
295
+
296
+ async def execute_model_async(
297
+ self,
298
+ execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
299
+ if self.parallel_worker_tasks is None:
300
+ # Start model execution loop running in the parallel workers
301
+ self.parallel_worker_tasks = asyncio.create_task(
302
+ self._start_worker_execution_loop())
303
+
304
+ # Only the driver worker returns the sampling results.
305
+ return await self._driver_execute_model_async(execute_model_req)
306
+
307
+ async def stop_remote_worker_execution_loop_async(self) -> None:
308
+ if self.parallel_worker_tasks is None:
309
+ return
310
+
311
+ await self._driver_execute_model_async()
312
+ parallel_worker_tasks = self.parallel_worker_tasks
313
+ self.parallel_worker_tasks = None
314
+ # Ensure that workers exit model loop cleanly
315
+ # (this will raise otherwise)
316
+ await parallel_worker_tasks
317
+
318
+ @abstractmethod
319
+ async def _driver_execute_model_async(
320
+ self,
321
+ execute_model_req: Optional[ExecuteModelRequest] = None,
322
+ ) -> List[SamplerOutput]:
323
+ """Execute the model asynchronously in the driver worker.
324
+
325
+ Passing None will cause the driver to stop the model execution
326
+ loop running in each of the remote workers.
327
+ """
328
+ raise NotImplementedError
329
+
330
+ @abstractmethod
331
+ async def _start_worker_execution_loop(self):
332
+ """Run execution loop on all workers. It guarantees all workers run
333
+ the loop or None of them is running the loop. Loop can be stopped by
334
+ `stop_remote_worker_execution_loop`.
335
+ The API is idempotent (guarantee only 1 loop run at any moment)."""
336
+ raise NotImplementedError
vllm/executor/gpu_executor.py
deleted
-145
@@ -1,145 +0,0 @@
1
-from typing import Any, Dict, List, Optional, Set, Tuple, Union
2
-
3
-from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4
-from vllm.logger import init_logger
5
-from vllm.lora.request import LoRARequest
6
-from vllm.model_executor.layers.sampler import SamplerOutput
7
-from vllm.prompt_adapter.request import PromptAdapterRequest
8
-from vllm.sequence import ExecuteModelRequest, PoolerOutput
9
-from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
10
- make_async)
11
-from vllm.worker.worker_base import WorkerWrapperBase
12
-
13
-logger = init_logger(__name__)
14
-
15
-
16
-def create_worker(**kwargs):
17
- vllm_config = kwargs.get("vllm_config")
18
- wrapper = WorkerWrapperBase(vllm_config=vllm_config)
19
- wrapper.init_worker(**kwargs)
20
- return wrapper.worker
21
-
22
-
23
-class GPUExecutor(ExecutorBase):
24
-
25
- uses_ray: bool = False
26
-
27
- def _init_executor(self) -> None:
28
- """Initialize the worker and load the model.
29
- """
30
- assert self.parallel_config.world_size == 1, (
31
- "GPUExecutor only supports single GPU.")
32
-
33
- self.driver_worker = self._create_worker()
34
- self.driver_worker.init_device()
35
- self.driver_worker.load_model()
36
-
37
- def _get_worker_kwargs(
38
- self,
39
- local_rank: int = 0,
40
- rank: int = 0,
41
- distributed_init_method: Optional[str] = None) -> Dict[str, Any]:
42
- """Return worker init args for a given rank."""
43
- if distributed_init_method is None:
44
- distributed_init_method = get_distributed_init_method(
45
- get_ip(), get_open_port())
46
- return dict(
47
- vllm_config=self.vllm_config,
48
- local_rank=local_rank,
49
- rank=rank,
50
- distributed_init_method=distributed_init_method,
51
- is_driver_worker=(not self.parallel_config)
52
- or (rank % self.parallel_config.tensor_parallel_size == 0),
53
- )
54
-
55
- def _create_worker(self,
56
- local_rank: int = 0,
57
- rank: int = 0,
58
- distributed_init_method: Optional[str] = None):
59
- return create_worker(**self._get_worker_kwargs(
60
- local_rank=local_rank,
61
- rank=rank,
62
- distributed_init_method=distributed_init_method))
63
-
64
- def determine_num_available_blocks(self) -> Tuple[int, int]:
65
- """Determine the number of available KV blocks by invoking the
66
- underlying worker.
67
- """
68
- return self.driver_worker.determine_num_available_blocks()
69
-
70
- def initialize_cache(self, num_gpu_blocks: int, num_cpu_blocks) -> None:
71
- """Initialize the KV cache by invoking the underlying worker.
72
- """
73
- # NOTE: This is logged in the executor because there can be >1 worker
74
- # with other executors. We could log in the engine level, but work
75
- # remains to abstract away the device for non-GPU configurations.
76
- logger.info("# GPU blocks: %d, # CPU blocks: %d", num_gpu_blocks,
77
- num_cpu_blocks)
78
- max_concurrency = (num_gpu_blocks * self.cache_config.block_size /
79
- self.model_config.max_model_len)
80
- logger.info("Maximum concurrency for %s tokens per request: %.2fx",
81
- self.model_config.max_model_len, max_concurrency)
82
-
83
- self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
84
-
85
- def execute_model(
86
- self, execute_model_req: ExecuteModelRequest
87
- ) -> Optional[List[Union[SamplerOutput, PoolerOutput]]]:
88
- output = self.driver_worker.execute_model(execute_model_req)
89
- return output
90
-
91
- def add_lora(self, lora_request: LoRARequest) -> bool:
92
- assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
93
- return self.driver_worker.add_lora(lora_request)
94
-
95
- def remove_lora(self, lora_id: int) -> bool:
96
- assert lora_id > 0, "lora_id must be greater than 0."
97
- return self.driver_worker.remove_lora(lora_id)
98
-
99
- def pin_lora(self, lora_id: int) -> bool:
100
- assert lora_id > 0, "lora_id must be greater than 0."
101
- return self.driver_worker.pin_lora(lora_id)
102
-
103
- def list_loras(self) -> Set[int]:
104
- return self.driver_worker.list_loras()
105
-
106
- def add_prompt_adapter(
107
- self, prompt_adapter_request: PromptAdapterRequest) -> bool:
108
- assert prompt_adapter_request.prompt_adapter_id > 0, \
109
- "prompt_adapter_id must be greater than 0."
110
- return self.driver_worker.add_prompt_adapter(prompt_adapter_request)
111
-
112
- def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
113
- assert prompt_adapter_id > 0, \
114
- "prompt_adapter_id must be greater than 0."
115
- return self.driver_worker.remove_prompt_adapter(prompt_adapter_id)
116
-
117
- def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
118
- assert prompt_adapter_id > 0, \
119
- "prompt_adapter_id must be greater than 0."
120
- return self.driver_worker.pin_prompt_adapter(prompt_adapter_id)
121
-
122
- def list_prompt_adapters(self) -> Set[int]:
123
- return self.driver_worker.list_prompt_adapters()
124
-
125
- def check_health(self) -> None:
126
- # GPUExecutor will always be healthy as long as
127
- # it's running.
128
- return
129
-
130
- def start_profile(self) -> None:
131
- self.driver_worker.start_profile()
132
-
133
- def stop_profile(self) -> None:
134
- self.driver_worker.stop_profile()
135
-
136
-
137
-class GPUExecutorAsync(GPUExecutor, ExecutorAsyncBase):
138
-
139
- async def execute_model_async(
140
- self,
141
- execute_model_req: ExecuteModelRequest,
142
- ) -> List[Union[SamplerOutput, PoolerOutput]]:
143
- output = await make_async(self.driver_worker.execute_model
144
- )(execute_model_req=execute_model_req)
145
- return output
vllm/executor/hpu_executor.py
deleted
-202
@@ -1,202 +0,0 @@
1
-###############################################################################
2
-# Copyright (C) 2024 Habana Labs, Ltd. an Intel Company
3
-###############################################################################
4
-
5
-import contextlib
6
-import os
7
-from typing import Any, Dict, List, Optional, Set, Tuple
8
-
9
-from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
10
-from vllm.logger import init_logger
11
-from vllm.lora.request import LoRARequest
12
-from vllm.model_executor.layers.sampler import SamplerOutput
13
-from vllm.prompt_adapter.request import PromptAdapterRequest
14
-from vllm.sequence import ExecuteModelRequest
15
-from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
16
- make_async)
17
-from vllm.worker.worker_base import WorkerWrapperBase
18
-
19
-logger = init_logger(__name__)
20
-
21
-
22
-class HPUExecutor(ExecutorBase):
23
-
24
- uses_ray: bool = False
25
-
26
- def _init_executor(self) -> None:
27
- """Initialize the worker and load the model."""
28
- self._init_worker()
29
-
30
- def _get_worker_kwargs(
31
- self,
32
- local_rank: int = 0,
33
- rank: int = 0,
34
- distributed_init_method: Optional[str] = None) -> Dict[str, Any]:
35
- """Return worker init args for a given rank."""
36
- if distributed_init_method is None:
37
- distributed_init_method = get_distributed_init_method(
38
- get_ip(), get_open_port())
39
- return dict(
40
- vllm_config=self.vllm_config,
41
- local_rank=local_rank,
42
- rank=rank,
43
- distributed_init_method=distributed_init_method,
44
- is_driver_worker=rank == 0,
45
- )
46
-
47
- def _create_worker(self,
48
- local_rank: int = 0,
49
- rank: int = 0,
50
- distributed_init_method: Optional[str] = None):
51
- wrapper = WorkerWrapperBase(vllm_config=self.vllm_config)
52
- wrapper.init_worker(**self._get_worker_kwargs(local_rank, rank,
53
- distributed_init_method))
54
- return wrapper.worker
55
-
56
- def _init_worker(self):
57
- assert self.parallel_config.world_size == 1, (
58
- "GPUExecutor only supports single GPU.")
59
-
60
- self.driver_worker = self._create_worker()
61
- self.driver_worker.init_device()
62
- self.driver_worker.load_model()
63
-
64
- def determine_num_available_blocks(self) -> Tuple[int, int]:
65
- """Determine the number of available KV blocks by invoking the
66
- underlying worker.
67
- """
68
- return self.driver_worker.determine_num_available_blocks()
69
-
70
- def initialize_cache(self, num_gpu_blocks: int, num_cpu_blocks) -> None:
71
- """Initialize the KV cache by invoking the underlying worker.
72
- """
73
- # NOTE: This is logged in the executor because there can be >1 worker
74
- # with other executors. We could log in the engine level, but work
75
- # remains to abstract away the device for non-GPU configurations.
76
- logger.info("# HPU blocks: %d, # CPU blocks: %d", num_gpu_blocks,
77
- num_cpu_blocks)
78
- from vllm_hpu_extension.profiler import HabanaMemoryProfiler
79
- with HabanaMemoryProfiler() as cache_init_m:
80
- self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
81
- msg = f"init_cache_engine took {cache_init_m.get_summary_string()}"
82
- logger.info(msg)
83
-
84
- def finish_measurements(self):
85
- self.driver_worker.finish_measurements()
86
-
87
- def execute_model(
88
- self,
89
- execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
90
- # VLLM_HPU_LOG_STEP_GRAPH_COMPILATION - will log graph compilations per engine step, only when there was any - highly recommended to use alongside PT_HPU_METRICS_GC_DETAILS! # noqa:E501
91
- # VLLM_HPU_LOG_STEP_GRAPH_COMPILATION_ALL - will log graph compilations per engine step, always, even if there were none # noqa:E501
92
- # VLLM_HPU_LOG_STEP_CPU_FALLBACKS - will log cpu fallbacks per engine step, only when there was any # noqa:E501
93
- # VLLM_HPU_LOG_STEP_CPU_FALLBACKS_ALL - will log cpu fallbacks per engine step, always, even if there were none # noqa:E501
94
- log_graph_compilation_all = os.environ.get(
95
- 'VLLM_HPU_LOG_STEP_GRAPH_COMPILATION_ALL', '0') != '0'
96
- log_graph_compilation = os.environ.get(
97
- 'VLLM_HPU_LOG_STEP_GRAPH_COMPILATION',
98
- '0') != '0' or log_graph_compilation_all
99
- log_cpu_fallbacks_all = os.environ.get(
100
- 'VLLM_HPU_LOG_STEP_CPU_FALLBACKS_ALL', '0') != '0'
101
- log_cpu_fallbacks = os.environ.get('VLLM_HPU_LOG_STEP_CPU_FALLBACKS',
102
- '0') != '0' or log_cpu_fallbacks_all
103
- if log_graph_compilation or log_cpu_fallbacks:
104
- from habana_frameworks.torch.hpu.metrics import metric_localcontext
105
- seq_group_metadata_list = execute_model_req.seq_group_metadata_list
106
- is_prompt = any([
107
- seq_group_metadata.is_prompt
108
- for seq_group_metadata in seq_group_metadata_list
109
- ])
110
- max_context_len = max([
111
- max([
112
- len(v.prompt_token_ids) + len(v.output_token_ids)
113
- for v in seq_group_metadata.seq_data.values()
114
- ]) for seq_group_metadata in seq_group_metadata_list
115
- ]) # whoa, that's some spicy stuff right here
116
- max_num_blocks = (
117
- (max_context_len - 1) // self.cache_config.block_size) + 1
118
- input_stats = (f'is_prompt: {is_prompt}, '
119
- f'num_seqs: {len(seq_group_metadata_list)}, '
120
- f'max_context_len: {max_context_len}, '
121
- f'max_num_blocks {max_num_blocks}')
122
- gc_ctx = metric_localcontext(
123
- "graph_compilation"
124
- ) if log_graph_compilation else contextlib.nullcontext()
125
- cpu_fallback_ctx = metric_localcontext(
126
- "cpu_fallback"
127
- ) if log_cpu_fallbacks else contextlib.nullcontext()
128
- with gc_ctx as gc_local_metric, \
129
- cpu_fallback_ctx as cpu_fallback_local_metric:
130
- output = self.driver_worker.execute_model(execute_model_req)
131
- if (log_graph_compilation and gc_local_metric.stats()[0][1] > 0
132
- ) or log_graph_compilation_all:
133
- msg = ("VLLM_HPU_STEP_GRAPH_COMPILATION: "
134
- f"{gc_local_metric.stats()}, {input_stats}")
135
- logger.warning(msg)
136
- if (log_cpu_fallbacks and cpu_fallback_local_metric.stats()[0][1] >
137
- 0) or log_cpu_fallbacks_all:
138
- msg = ("VLLM_HPU_STEP_CPU_FALLBACK: "
139
- f"{cpu_fallback_local_metric.stats()}, {input_stats}")
140
- logger.warning(msg)
141
-
142
- return output
143
-
144
- output = self.driver_worker.execute_model(execute_model_req)
145
- return output
146
-
147
- def add_lora(self, lora_request: LoRARequest) -> bool:
148
- assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
149
- return self.driver_worker.add_lora(lora_request)
150
-
151
- def remove_lora(self, lora_id: int) -> bool:
152
- assert lora_id > 0, "lora_id must be greater than 0."
153
- return self.driver_worker.remove_lora(lora_id)
154
-
155
- def pin_lora(self, lora_id: int) -> bool:
156
- assert lora_id > 0, "lora_id must be greater than 0."
157
- return self.driver_worker.pin_lora(lora_id)
158
-
159
- def list_loras(self) -> Set[int]:
160
- return self.driver_worker.list_loras()
161
-
162
- def add_prompt_adapter(
163
- self, prompt_adapter_request: PromptAdapterRequest) -> bool:
164
- raise NotImplementedError(
165
- "Prompt Adapter is not implemented for HPU backend.")
166
-
167
- def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
168
- raise NotImplementedError(
169
- "Prompt Adapter is not implemented for HPU backend.")
170
-
171
- def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
172
- raise NotImplementedError(
173
- "Prompt Adapter is not implemented for HPU backend.")
174
-
175
- def list_prompt_adapters(self) -> Set[int]:
176
- raise NotImplementedError(
177
- "Prompt Adapter is not implemented for HPU backend.")
178
-
179
- def check_health(self) -> None:
180
- # GPUExecutor will always be healthy as long as
181
- # it's running.
182
- return
183
-
184
- def start_profile(self) -> None:
185
- self.driver_worker.start_profile()
186
-
187
- def stop_profile(self) -> None:
188
- self.driver_worker.stop_profile()
189
-
190
- def shutdown(self) -> None:
191
- self.driver_worker.shutdown_inc()
192
-
193
-
194
-class HPUExecutorAsync(HPUExecutor, ExecutorAsyncBase):
195
-
196
- async def execute_model_async(
197
- self,
198
- execute_model_req: ExecuteModelRequest,
199
- ) -> List[SamplerOutput]:
200
- output = await make_async(self.driver_worker.execute_model
201
- )(execute_model_req=execute_model_req, )
202
- return output
vllm/executor/mp_distributed_executor.py
renamed
+29
-52
@@ -1,32 +1,26 @@
1
import asyncio
2
-import os
3
-from functools import partial
2
from typing import Any, List, Optional
3
6
-from vllm.executor.distributed_gpu_executor import ( # yapf: disable
7
- DistributedGPUExecutor, DistributedGPUExecutorAsync)
8
-from vllm.executor.gpu_executor import create_worker
4
+from vllm.executor.executor_base import DistributedExecutorBase
5
from vllm.executor.multiproc_worker_utils import (
6
ProcessWorkerWrapper, ResultHandler, WorkerMonitor,
7
set_multiprocessing_worker_envs)
8
from vllm.logger import init_logger
9
from vllm.model_executor.layers.sampler import SamplerOutput
10
from vllm.sequence import ExecuteModelRequest
15
-from vllm.utils import (_run_task_with_lock, cuda_device_count_stateless,
16
- get_distributed_init_method, get_open_port, make_async,
17
- update_environment_variables)
11
+from vllm.utils import (_run_task_with_lock, get_distributed_init_method,
12
+ get_ip, get_open_port, make_async)
13
+from vllm.worker.worker_base import WorkerWrapperBase
14
15
logger = init_logger(__name__)
16
17
22
-class MultiprocessingGPUExecutor(DistributedGPUExecutor):
23
- """Python multiprocessing-based multi-GPU executor"""
18
+class MultiprocessingDistributedExecutor(DistributedExecutorBase):
19
+ """Python multiprocessing-based distributed executor"""
20
21
uses_ray: bool = False
22
23
def _init_executor(self) -> None:
28
- self._check_executor_parameters()
29
-
24
# Create the parallel GPU workers.
25
world_size = self.parallel_config.world_size
26
tensor_parallel_size = self.parallel_config.tensor_parallel_size
@@ -55,15 +49,9 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
49
else:
50
result_handler = ResultHandler()
51
for rank in range(1, world_size):
58
- worker = ProcessWorkerWrapper(
59
- result_handler,
60
- partial(
61
- create_worker,
62
- **self._get_worker_kwargs(
63
- rank=rank,
64
- local_rank=rank,
65
- distributed_init_method=distributed_init_method,
66
- )))
52
+ worker = ProcessWorkerWrapper(result_handler,
53
+ WorkerWrapperBase,
54
+ self.vllm_config, rank)
55
self.workers.append(worker)
56
if rank % tensor_parallel_size == 0:
57
self.tp_driver_workers.append(worker)
@@ -77,32 +65,30 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
65
# Set up signal handlers to shutdown the executor cleanly
66
# sometimes gc does not work well
67
80
- self.driver_worker = self._create_worker(
81
- distributed_init_method=distributed_init_method)
68
+ self.driver_worker = WorkerWrapperBase(self.vllm_config, 0)
69
+
70
+ all_kwargs = []
71
+ distributed_init_method = get_distributed_init_method(
72
+ get_ip(), get_open_port())
73
+ for i in range(world_size):
74
+ local_rank = i
75
+ rank = i
76
+ kwargs = dict(
77
+ vllm_config=self.vllm_config,
78
+ local_rank=local_rank,
79
+ rank=rank,
80
+ distributed_init_method=distributed_init_method,
81
+ is_driver_worker=(not self.parallel_config)
82
+ or (rank % self.parallel_config.tensor_parallel_size == 0),
83
+ )
84
+ all_kwargs.append(kwargs)
85
+ self._run_workers("init_worker", all_kwargs)
86
self._run_workers("init_device")
87
self._run_workers("load_model",
88
max_concurrent_workers=self.parallel_config.
89
max_parallel_loading_workers)
86
-
87
- def _check_executor_parameters(self):
88
- world_size = self.parallel_config.world_size
89
- tensor_parallel_size = self.parallel_config.tensor_parallel_size
90
-
91
- # Set CUDA_VISIBLE_DEVICES for the driver, inherited by workers
92
- if "CUDA_VISIBLE_DEVICES" not in os.environ:
93
- update_environment_variables({
94
- "CUDA_VISIBLE_DEVICES": (",".join(map(str, range(world_size))))
95
- })
96
-
97
- cuda_device_count = cuda_device_count_stateless()
98
- # Use confusing message for more common TP-only case.
99
- assert tensor_parallel_size <= cuda_device_count, (
100
- f"please set tensor_parallel_size ({tensor_parallel_size}) "
101
- f"to less than max local gpu count ({cuda_device_count})")
102
-
103
- assert world_size <= cuda_device_count, (
104
- f"please ensure that world_size ({world_size}) "
105
- f"is less than than max local gpu count ({cuda_device_count})")
90
+ self.driver_exec_model = make_async(self.driver_worker.execute_model)
91
+ self.pp_locks: Optional[List[asyncio.Lock]] = None
92
93
def shutdown(self):
94
if (worker_monitor := getattr(self, "worker_monitor",
@@ -172,15 +158,6 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
158
for result in parallel_worker_tasks:
159
result.get()
160
175
-
176
-class MultiprocessingGPUExecutorAsync(MultiprocessingGPUExecutor,
177
- DistributedGPUExecutorAsync):
178
-
179
- def __init__(self, *args, **kwargs):
180
- super().__init__(*args, **kwargs)
181
- self.driver_exec_model = make_async(self.driver_worker.execute_model)
182
- self.pp_locks: Optional[List[asyncio.Lock]] = None
183
-
161
async def _driver_execute_model_async(
162
self,
163
execute_model_req: Optional[ExecuteModelRequest] = None
vllm/executor/multiproc_worker_utils.py
+9
-3
@@ -12,6 +12,7 @@ from typing import (Any, Callable, Dict, Generic, List, Optional, TextIO,
12
13
import torch
14
15
+from vllm.config import VllmConfig
16
from vllm.logger import init_logger
17
from vllm.triton_utils.importing import HAS_TRITON
18
from vllm.utils import _check_multiproc_method, get_mp_context
@@ -147,7 +148,8 @@ class ProcessWorkerWrapper:
148
for handling single-node multi-GPU tensor parallel."""
149
150
def __init__(self, result_handler: ResultHandler,
150
- worker_factory: Callable[[], Any]) -> None:
151
+ worker_factory: Callable[[VllmConfig, int], Any],
152
+ vllm_config: VllmConfig, rank: int) -> None:
153
self.mp = get_mp_context()
154
self._task_queue = self.mp.Queue()
155
self.result_queue = result_handler.result_queue
@@ -159,6 +161,8 @@ class ProcessWorkerWrapper:
161
worker_factory=worker_factory,
162
task_queue=self._task_queue,
163
result_queue=self.result_queue,
164
+ vllm_config=vllm_config,
165
+ rank=rank,
166
),
167
daemon=True)
168
@@ -199,9 +203,11 @@ class ProcessWorkerWrapper:
203
204
205
def _run_worker_process(
202
- worker_factory: Callable[[], Any],
206
+ worker_factory: Callable[[VllmConfig, int], Any],
207
task_queue: Queue,
208
result_queue: Queue,
209
+ vllm_config: VllmConfig,
210
+ rank: int,
211
) -> None:
212
"""Worker process event loop"""
213
@@ -212,7 +218,7 @@ def _run_worker_process(
218
_add_prefix(sys.stderr, process_name, pid)
219
220
# Initialize worker
215
- worker = worker_factory()
221
+ worker = worker_factory(vllm_config, rank)
222
del worker_factory
223
224
# Accept tasks from the engine in task_queue
vllm/executor/multiproc_xpu_executor.py
deleted
-26
@@ -1,26 +0,0 @@
1
-import vllm.envs as envs
2
-from vllm.executor.multiproc_gpu_executor import (
3
- MultiprocessingGPUExecutor, MultiprocessingGPUExecutorAsync)
4
-from vllm.executor.xpu_executor import XPUExecutor
5
-from vllm.logger import init_logger
6
-from vllm.utils import make_async
7
-
8
-logger = init_logger(__name__)
9
-
10
-
11
-class MultiprocessingXPUExecutor(MultiprocessingGPUExecutor, XPUExecutor):
12
- """Python multiprocessing-based multi-XPU executor"""
13
-
14
- def _check_executor_parameters(self):
15
- mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD
16
- if mp_method != "spawn":
17
- raise RuntimeError(
18
- "XPU multiprocess executor only support spawn as mp method")
19
-
20
-
21
-class MultiprocessingXPUExecutorAsync(MultiprocessingXPUExecutor,
22
- MultiprocessingGPUExecutorAsync):
23
-
24
- def __init__(self, *args, **kwargs):
25
- super().__init__(*args, **kwargs)
26
- self.driver_exec_model = make_async(self.driver_worker.execute_model)
vllm/executor/neuron_executor.py
deleted
-114
@@ -1,114 +0,0 @@
1
-from typing import List, Set, Tuple
2
-
3
-from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4
-from vllm.logger import init_logger
5
-from vllm.lora.request import LoRARequest
6
-from vllm.model_executor.layers.sampler import SamplerOutput
7
-from vllm.sequence import ExecuteModelRequest
8
-from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
9
- make_async)
10
-from vllm.worker.worker_base import WorkerWrapperBase
11
-
12
-logger = init_logger(__name__)
13
-
14
-
15
-class NeuronExecutor(ExecutorBase):
16
-
17
- uses_ray: bool = False
18
-
19
- def _init_executor(self) -> None:
20
- assert (self.lora_config is
21
- None), "LoRA is not supported for Neuron backend."
22
- assert (not self.speculative_config
23
- ), "Speculative decoding not yet supported for Neuron backend."
24
-
25
- # Instantiate the worker and load the model to the device.
26
- self._init_worker()
27
-
28
- def _init_worker(self):
29
- wrapper = WorkerWrapperBase(vllm_config=self.vllm_config)
30
- distributed_init_method = get_distributed_init_method(
31
- get_ip(), get_open_port())
32
- wrapper.init_worker(
33
- vllm_config=self.vllm_config,
34
- local_rank=0,
35
- rank=0,
36
- distributed_init_method=distributed_init_method,
37
- )
38
- self.driver_worker = wrapper.worker
39
- self.driver_worker.init_device()
40
- self.driver_worker.load_model()
41
-
42
- def determine_num_available_blocks(self) -> Tuple[int, int]:
43
- """Determine the number of available KV blocks by invoking the
44
- underlying worker.
45
- """
46
- return self.driver_worker.determine_num_available_blocks()
47
-
48
- def initialize_cache(self, num_gpu_blocks: int,
49
- num_cpu_blocks: int) -> None:
50
- """Initialize the KV cache by invoking the underlying worker.
51
- """
52
- self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
53
-
54
- def execute_model(
55
- self,
56
- execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
57
- assert (not execute_model_req.blocks_to_swap_in
58
- and not execute_model_req.blocks_to_swap_out
59
- and not execute_model_req.blocks_to_copy), (
60
- "Cache operations are not supported for Neuron backend.")
61
- assert execute_model_req.num_lookahead_slots == 0, (
62
- "lookahead not supported for Neuron backend.")
63
-
64
- output = self.driver_worker.execute_model(execute_model_req)
65
- return output
66
-
67
- def add_lora(self, lora_request: LoRARequest) -> bool:
68
- return self.driver_worker.add_lora(lora_request)
69
-
70
- def remove_lora(self, lora_id: int) -> bool:
71
- return self.driver_worker.remove_lora(lora_id)
72
-
73
- def pin_lora(self, lora_id: int) -> bool:
74
- return self.driver_worker.pin_lora(lora_id)
75
-
76
- def list_loras(self) -> Set[int]:
77
- return self.driver_worker.list_loras()
78
-
79
- def add_prompt_adapter(self, prompt_adapter_request) -> bool:
80
- raise NotImplementedError(
81
- "Soft prompt is currently not supported by the Neuron backend.")
82
-
83
- def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
84
- raise NotImplementedError(
85
- "Soft prompt is currently not supported by the Neuron backend.")
86
-
87
- def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
88
- raise NotImplementedError(
89
- "Soft prompt is currently not supported by the Neuron backend.")
90
-
91
- def list_prompt_adapters(self) -> Set[int]:
92
- raise NotImplementedError(
93
- "Soft prompt is currently not supported by the Neuron backend.")
94
-
95
- def check_health(self) -> None:
96
- # NeuronExecutor will always be healthy as long as
97
- # it's running.
98
- return
99
-
100
-
101
-class NeuronExecutorAsync(NeuronExecutor, ExecutorAsyncBase):
102
-
103
- async def execute_model_async(
104
- self,
105
- execute_model_req: ExecuteModelRequest,
106
- ) -> List[SamplerOutput]:
107
- output = await make_async(self.driver_worker.execute_model
108
- )(execute_model_req=execute_model_req, )
109
- return output
110
-
111
- async def check_health_async(self) -> None:
112
- # NeuronExecutor will always be healthy as long as
113
- # it's running.
114
- return
vllm/executor/openvino_executor.py
deleted
-125
@@ -1,125 +0,0 @@
1
-from typing import List, Set, Tuple
2
-
3
-import openvino as ov
4
-
5
-import vllm.envs as envs
6
-from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
7
-from vllm.logger import init_logger
8
-from vllm.lora.request import LoRARequest
9
-from vllm.model_executor.layers.sampler import SamplerOutput
10
-from vllm.platforms import current_platform
11
-from vllm.sequence import ExecuteModelRequest
12
-from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
13
- make_async)
14
-from vllm.worker.worker_base import WorkerWrapperBase
15
-
16
-logger = init_logger(__name__)
17
-
18
-
19
-class OpenVINOExecutor(ExecutorBase):
20
-
21
- uses_ray: bool = False
22
-
23
- def _init_executor(self) -> None:
24
- assert self.device_config.device_type == "openvino"
25
- assert self.lora_config is None, "OpenVINO backend doesn't support LoRA"
26
- assert current_platform.is_openvino_cpu() or \
27
- current_platform.is_openvino_gpu(), \
28
- "OpenVINO backend supports only CPU and GPU devices"
29
-
30
- # Instantiate the worker and load the model to CPU.
31
- self._init_worker()
32
-
33
- def _init_worker(self):
34
-
35
- wrapper = WorkerWrapperBase(vllm_config=self.vllm_config)
36
-
37
- distributed_init_method = get_distributed_init_method(
38
- get_ip(), get_open_port())
39
- wrapper.init_worker(
40
- ov_core=ov.Core(),
41
- vllm_config=self.vllm_config,
42
- local_rank=0,
43
- rank=0,
44
- distributed_init_method=distributed_init_method,
45
- kv_cache_dtype=self.cache_config.cache_dtype,
46
- is_driver_worker=True,
47
- )
48
- self.driver_worker = wrapper.worker
49
- self.driver_worker.init_device()
50
- self.driver_worker.load_model()
51
-
52
- def determine_num_available_blocks(self) -> Tuple[int, int]:
53
- """Determine the number of available KV blocks by invoking the
54
- underlying worker.
55
- """
56
- return self.driver_worker.determine_num_available_blocks()
57
-
58
- def initialize_cache(self, num_gpu_blocks: int,
59
- num_cpu_blocks: int) -> None:
60
- """Initialize the KV cache by invoking the underlying worker."""
61
- # NOTE: We log here to avoid multiple logs when number of workers is
62
- # greater than one. We could log in the engine, but not all executors
63
- # have GPUs.
64
- # NOTE: In case of a CPU device, `cpu block` for OpenVINO backend
65
- # is located on CPU memory but is referred as `gpu block`.
66
- # Because we want to reuse the existing block management procedure.
67
- device_blocks = num_gpu_blocks
68
- swap_blocks = num_cpu_blocks
69
- logger.info("OpenVINO %s: # device blocks: %d; # swap blocks: %d",
70
- envs.VLLM_OPENVINO_DEVICE, device_blocks, swap_blocks)
71
- self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
72
-
73
- def execute_model(
74
- self,
75
- execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
76
- output = self.driver_worker.execute_model(execute_model_req)
77
- return output
78
-
79
- def add_lora(self, lora_request: LoRARequest) -> bool:
80
- return self.driver_worker.add_lora(lora_request)
81
-
82
- def remove_lora(self, lora_id: int) -> bool:
83
- return self.driver_worker.remove_lora(lora_id)
84
-
85
- def pin_lora(self, lora_id: int) -> bool:
86
- return self.driver_worker.pin_lora(lora_id)
87
-
88
- def list_loras(self) -> Set[int]:
89
- return self.driver_worker.list_loras()
90
-
91
- def add_prompt_adapter(self, prompt_adapter_request) -> bool:
92
- raise NotImplementedError(
93
- "Soft prompt is currently not supported by the OPENVINO backend.")
94
-
95
- def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
96
- raise NotImplementedError(
97
- "Soft prompt is currently not supported by the OPENVINO backend.")
98
-
99
- def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
100
- raise NotImplementedError(
101
- "Soft prompt is currently not supported by the OPENVINO backend.")
102
-
103
- def list_prompt_adapters(self) -> Set[int]:
104
- raise NotImplementedError(
105
- "Soft prompt is currently not supported by the OPENVINO backend.")
106
-
107
- def check_health(self) -> None:
108
- # OpenVINOExecutor will always be healthy as long as
109
- # it's running.
110
- return
111
-
112
-
113
-class OpenVINOExecutorAsync(OpenVINOExecutor, ExecutorAsyncBase):
114
-
115
- async def execute_model_async(
116
- self,
117
- execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
118
- output = await make_async(self.driver_worker.execute_model
119
- )(execute_model_req=execute_model_req, )
120
- return output
121
-
122
- async def check_health_async(self) -> None:
123
- # OpenVINOExecutor will always be healthy as long as
124
- # it's running.
125
- return
vllm/executor/ray_distributed_executor.py
renamed
+144
-96
@@ -1,24 +1,29 @@
1
import asyncio
2
import os
3
from collections import defaultdict
4
-from itertools import islice, repeat
5
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
4
+from dataclasses import dataclass
5
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
6
7
import msgspec
8
9
import vllm.envs as envs
10
-from vllm.executor.distributed_gpu_executor import ( # yapf: disable
11
- DistributedGPUExecutor, DistributedGPUExecutorAsync)
10
+from vllm.executor.executor_base import (
11
+ DistributedExecutorBase) # yapf: disable
12
from vllm.executor.msgspec_utils import encode_hook
13
-from vllm.executor.ray_utils import RayWorkerWrapper, ray
13
+from vllm.executor.ray_utils import (RayWorkerWrapper, initialize_ray_cluster,
14
+ ray)
15
from vllm.logger import init_logger
16
from vllm.model_executor.layers.sampler import SamplerOutput
17
+from vllm.platforms import current_platform
18
from vllm.sequence import ExecuteModelRequest
19
from vllm.utils import (_run_task_with_lock, get_distributed_init_method,
20
get_ip, get_open_port, make_async)
21
22
if ray is not None:
23
+ from ray.actor import ActorHandle
24
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
25
+else:
26
+ ActorHandle = None
27
28
if TYPE_CHECKING:
29
from ray.util.placement_group import PlacementGroup
@@ -26,12 +31,29 @@ if TYPE_CHECKING:
31
logger = init_logger(__name__)
32
33
29
-class RayGPUExecutor(DistributedGPUExecutor):
34
+@dataclass
35
+class RayWorkerMetaData:
36
+ """
37
+ Metadata for a Ray worker.
38
+ The order of ray worker creation can be random,
39
+ and we need to reset the rank after creating all workers.
40
+ """
41
+ worker: ActorHandle
42
+ created_rank: int
43
+ adjusted_rank: int = -1
44
+ ip: str = ""
45
+
46
+
47
+class RayDistributedExecutor(DistributedExecutorBase):
48
49
uses_ray: bool = True
50
51
def _init_executor(self) -> None:
52
self.forward_dag: Optional[ray.dag.CompiledDAG] = None
53
+ if envs.VLLM_USE_V1:
54
+ # v1 always uses the compiled DAG and SPMD worker.
55
+ os.environ["VLLM_USE_RAY_SPMD_WORKER"] = "1"
56
+ os.environ["VLLM_USE_RAY_COMPILED_DAG"] = "1"
57
# If the env var is set, it uses the Ray's compiled DAG API
58
# which optimizes the control plane overhead.
59
# Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
@@ -53,6 +75,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
75
"VLLM_USE_RAY_COMPILED_DAG=1")
76
77
assert self.uses_ray
78
+ initialize_ray_cluster(self.parallel_config)
79
placement_group = self.parallel_config.placement_group
80
81
# Disable Ray usage stats collection.
@@ -66,6 +89,13 @@ class RayGPUExecutor(DistributedGPUExecutor):
89
self.input_encoder = msgspec.msgpack.Encoder(enc_hook=encode_hook)
90
self.output_decoder = msgspec.msgpack.Decoder(
91
Optional[List[SamplerOutput]])
92
+ self.use_v1 = envs.VLLM_USE_V1
93
+
94
+ self.pp_locks: Optional[List[asyncio.Lock]] = None
95
+ self.use_ray_spmd_worker = envs.VLLM_USE_RAY_SPMD_WORKER
96
+ if not self.use_ray_compiled_dag:
97
+ self.driver_exec_method = make_async(
98
+ self.driver_worker.execute_method)
99
100
def shutdown(self) -> None:
101
if hasattr(self, "forward_dag") and self.forward_dag is not None:
@@ -123,9 +153,10 @@ class RayGPUExecutor(DistributedGPUExecutor):
153
154
# Create the workers.
155
driver_ip = get_ip()
126
- workers = []
156
+ rank = 0
157
+ worker_metadata: List[RayWorkerMetaData] = []
158
for bundle_id, bundle in enumerate(placement_group.bundle_specs):
128
- if not bundle.get("GPU", 0):
159
+ if not bundle.get(current_platform.ray_device_key, 0):
160
continue
161
scheduling_strategy = PlacementGroupSchedulingStrategy(
162
placement_group=placement_group,
@@ -133,38 +164,51 @@ class RayGPUExecutor(DistributedGPUExecutor):
164
placement_group_bundle_index=bundle_id,
165
)
166
136
- worker = ray.remote(
137
- num_cpus=0,
138
- num_gpus=num_gpus,
139
- scheduling_strategy=scheduling_strategy,
140
- **ray_remote_kwargs,
141
- )(RayWorkerWrapper).remote(vllm_config=self.vllm_config)
142
- workers.append(worker)
143
-
144
- worker_ip_refs = [
145
- worker.get_node_ip.remote() # type: ignore[attr-defined]
146
- for worker in workers
147
- ]
148
- worker_ips = ray.get(worker_ip_refs)
167
+ if current_platform.ray_device_key == "GPU":
168
+ # NV+AMD GPUs, and Intel XPUs
169
+ worker = ray.remote(
170
+ num_cpus=0,
171
+ num_gpus=num_gpus,
172
+ scheduling_strategy=scheduling_strategy,
173
+ **ray_remote_kwargs,
174
+ )(RayWorkerWrapper).remote(vllm_config=self.vllm_config,
175
+ rank=rank)
176
+ else:
177
+ worker = ray.remote(
178
+ num_cpus=0,
179
+ num_gpus=0,
180
+ resources={current_platform.ray_device_key: num_gpus},
181
+ scheduling_strategy=scheduling_strategy,
182
+ **ray_remote_kwargs,
183
+ )(RayWorkerWrapper).remote(vllm_config=self.vllm_config,
184
+ rank=rank)
185
+ worker_metadata.append(
186
+ RayWorkerMetaData(worker=worker, created_rank=rank))
187
+ rank += 1
188
+
189
+ worker_ips = ray.get([
190
+ each.worker.get_node_ip.remote() # type: ignore[attr-defined]
191
+ for each in worker_metadata
192
+ ])
193
+
194
+ for each, ip in zip(worker_metadata, worker_ips):
195
+ each.ip = ip
196
197
if not self.use_ray_spmd_worker:
151
- for i in range(len(workers)):
152
- worker = workers[i]
153
- worker_ip = worker_ips[i]
198
+ for i, each in enumerate(worker_metadata):
199
+ # find and remove the dummy worker from the list
200
+ worker = each.worker
201
+ worker_ip = each.ip
202
if self.driver_dummy_worker is None and worker_ip == driver_ip:
203
# If the worker is on the same node as the driver, we use it
204
# as the resource holder for the driver process.
205
self.driver_dummy_worker = worker
206
self.driver_worker = RayWorkerWrapper(
159
- vllm_config=self.vllm_config)
160
- workers.pop(i)
161
- worker_ips.pop(i)
162
- self.workers = workers
207
+ vllm_config=self.vllm_config, rank=0)
208
+ worker_metadata.pop(i)
209
break
164
- else:
165
- self.workers = workers
210
167
- logger.debug("workers: %s", self.workers)
211
+ logger.debug("workers: %s", worker_metadata)
212
logger.debug("driver_dummy_worker: %s", self.driver_dummy_worker)
213
if not self.use_ray_spmd_worker and self.driver_dummy_worker is None:
214
raise ValueError(
@@ -176,9 +220,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
220
for ip in worker_ips:
221
ip_counts[ip] = ip_counts.get(ip, 0) + 1
222
179
- worker_to_ip = dict(zip(self.workers, worker_ips))
180
-
181
- def sort_by_driver_then_worker_ip(worker):
223
+ def sort_by_driver_then_worker_ip(item: RayWorkerMetaData):
224
"""
225
Sort the workers based on 3 properties:
226
1. If the worker is on the same node as the driver (vllm engine),
@@ -188,13 +230,23 @@ class RayGPUExecutor(DistributedGPUExecutor):
230
3. Finally, if the work is on a node with smaller IP address, it
231
should be placed first.
232
"""
191
- ip = worker_to_ip[worker]
192
- return (ip != driver_ip, ip_counts[ip], ip)
233
+ ip = item.ip
234
+ return (0 if ip == driver_ip else 1, ip_counts[ip], ip)
235
236
# After sorting, the workers on the same node will be
237
# close to each other, and the workers on the driver
238
# node will be placed first.
197
- self.workers = sorted(self.workers, key=sort_by_driver_then_worker_ip)
239
+ sorted_worker_metadata = sorted(worker_metadata,
240
+ key=sort_by_driver_then_worker_ip)
241
+ start_rank = 0 if self.use_ray_spmd_worker else 1
242
+ for i, item in enumerate(sorted_worker_metadata):
243
+ item.adjusted_rank = i + start_rank
244
+ self.workers = [item.worker for item in sorted_worker_metadata]
245
+ rerank_mapping = {
246
+ item.created_rank: item.adjusted_rank
247
+ for item in sorted_worker_metadata
248
+ }
249
+ self._run_workers("adjust_rank", rerank_mapping)
250
251
# Get the set of GPU IDs used on each node.
252
worker_node_and_gpu_ids = []
@@ -235,21 +287,29 @@ class RayGPUExecutor(DistributedGPUExecutor):
287
" each node.")
288
289
# Set environment variables for the driver and workers.
238
- all_args_to_update_environment_variables = [({
239
- "CUDA_VISIBLE_DEVICES":
290
+ all_args_to_update_environment_variables = [{
291
+ current_platform.device_control_env_var:
292
",".join(map(str, node_gpus[node_id])),
241
- "VLLM_TRACE_FUNCTION":
242
- str(envs.VLLM_TRACE_FUNCTION),
243
- **({
244
- "VLLM_ATTENTION_BACKEND": envs.VLLM_ATTENTION_BACKEND
245
- } if envs.VLLM_ATTENTION_BACKEND is not None else {})
246
- }, ) for (node_id, _) in worker_node_and_gpu_ids]
293
+ } for (node_id, _) in worker_node_and_gpu_ids]
294
+
295
+ for args in all_args_to_update_environment_variables:
296
+ # some carry-over env vars from the driver
297
+ # TODO: refactor platform-specific env vars
298
+ for name in [
299
+ "VLLM_ATTENTION_BACKEND",
300
+ "TPU_CHIPS_PER_HOST_BOUNDS",
301
+ "TPU_HOST_BOUNDS",
302
+ "VLLM_USE_V1",
303
+ "VLLM_TRACE_FUNCTION",
304
+ ]:
305
+ if name in os.environ:
306
+ args[name] = os.environ[name]
307
308
self._env_vars_for_all_workers = (
309
all_args_to_update_environment_variables)
310
311
self._run_workers("update_environment_variables",
252
- all_args=self._get_env_vars_to_be_updated())
312
+ self._get_env_vars_to_be_updated())
313
314
if len(node_gpus) == 1:
315
# in single node case, we don't need to get the IP address.
@@ -265,14 +325,19 @@ class RayGPUExecutor(DistributedGPUExecutor):
325
driver_ip, get_open_port())
326
327
# Initialize the actual workers inside worker wrapper.
268
- init_worker_all_kwargs = [
269
- self._get_worker_kwargs(
270
- local_rank=node_workers[node_id].index(rank),
328
+ all_kwargs = []
329
+ for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids):
330
+ local_rank = node_workers[node_id].index(rank)
331
+ kwargs = dict(
332
+ vllm_config=self.vllm_config,
333
+ local_rank=local_rank,
334
rank=rank,
335
distributed_init_method=distributed_init_method,
273
- ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids)
274
- ]
275
- self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
336
+ is_driver_worker=(not self.parallel_config)
337
+ or (rank % self.parallel_config.tensor_parallel_size == 0),
338
+ )
339
+ all_kwargs.append(kwargs)
340
+ self._run_workers("init_worker", all_kwargs)
341
342
self._run_workers("init_device")
343
self._run_workers("load_model",
@@ -332,9 +397,15 @@ class RayGPUExecutor(DistributedGPUExecutor):
397
if self.forward_dag is None:
398
self.forward_dag = self._compiled_ray_dag(enable_asyncio=False)
399
335
- serialized_data = self.input_encoder.encode(execute_model_req)
400
+ if self.use_v1:
401
+ serialized_data = execute_model_req
402
+ else:
403
+ serialized_data = self.input_encoder.encode(execute_model_req)
404
outputs = ray.get(self.forward_dag.execute(serialized_data))
337
- output = self.output_decoder.decode(outputs[0])
405
+ if self.use_v1:
406
+ output = outputs[0]
407
+ else:
408
+ output = self.output_decoder.decode(outputs[0])
409
return output
410
411
def _run_workers(
@@ -342,8 +413,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
413
method: str,
414
*args,
415
async_run_tensor_parallel_workers_only: bool = False,
345
- all_args: Optional[List[Tuple[Any, ...]]] = None,
346
- all_kwargs: Optional[List[Dict[str, Any]]] = None,
416
max_concurrent_workers: Optional[int] = None,
417
**kwargs,
418
) -> Any:
@@ -356,8 +425,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
425
It will also be run asynchronously and return a list of futures
426
rather than blocking on the results.
427
- args/kwargs: All workers share the same args/kwargs
359
- - all_args/all_kwargs: args/kwargs for each worker are specified
360
- individually
428
"""
429
if self.use_ray_spmd_worker:
430
assert not async_run_tensor_parallel_workers_only, (
@@ -368,26 +435,13 @@ class RayGPUExecutor(DistributedGPUExecutor):
435
raise NotImplementedError(
436
"max_concurrent_workers is not supported yet.")
437
371
- count = len(self.workers) if not \
372
- async_run_tensor_parallel_workers_only \
373
- else len(self.non_driver_workers)
374
- # If using SPMD worker, all workers are the same, so we should execute
375
- # the args on all workers. Otherwise, we skip the first worker's args
376
- # because those args will go to the driver worker.
377
- first_worker_args_index: int = 0 if self.use_ray_spmd_worker else 1
378
- all_worker_args = repeat(args, count) if all_args is None \
379
- else islice(all_args, first_worker_args_index, None)
380
- all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
381
- else islice(all_kwargs, first_worker_args_index, None)
382
-
438
# Start the ray workers first.
439
ray_workers = self.workers
440
if async_run_tensor_parallel_workers_only:
441
ray_workers = self.non_driver_workers
442
ray_worker_outputs = [
388
- worker.execute_method.remote(method, *worker_args, **worker_kwargs)
389
- for (worker, worker_args, worker_kwargs
390
- ) in zip(ray_workers, all_worker_args, all_worker_kwargs)
443
+ worker.execute_method.remote(method, *args, **kwargs)
444
+ for worker in ray_workers
445
]
446
447
if async_run_tensor_parallel_workers_only:
@@ -399,13 +453,9 @@ class RayGPUExecutor(DistributedGPUExecutor):
453
# so we only explicitly execute on the driver worker if using a
454
# non-SPMD worker class.
455
if not self.use_ray_spmd_worker:
402
- driver_args = args if all_args is None else all_args[0]
403
- driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
404
-
456
# Start the driver worker after all the ray workers.
457
driver_worker_output = [
407
- self.driver_worker.execute_method(method, *driver_args,
408
- **driver_kwargs)
458
+ self.driver_worker.execute_method(method, *args, **kwargs)
459
]
460
461
# Get the results of the ray workers.
@@ -467,11 +517,18 @@ class RayGPUExecutor(DistributedGPUExecutor):
517
for pp_rank, tp_group in enumerate(self.pp_tp_workers):
518
# Each PP worker takes in the output of the previous PP worker,
519
# and the TP group executes in SPMD fashion.
470
- outputs = [
471
- worker.execute_model_spmd.
472
- bind( # type: ignore[attr-defined]
473
- outputs[i]) for i, worker in enumerate(tp_group)
474
- ]
520
+ if self.use_v1:
521
+ outputs = [
522
+ worker.execute_model.
523
+ bind( # type: ignore[attr-defined]
524
+ outputs[i]) for i, worker in enumerate(tp_group)
525
+ ]
526
+ else:
527
+ outputs = [
528
+ worker.execute_model_spmd.
529
+ bind( # type: ignore[attr-defined]
530
+ outputs[i]) for i, worker in enumerate(tp_group)
531
+ ]
532
533
last_pp_rank = len(self.pp_tp_workers) - 1
534
if pp_rank < last_pp_rank:
@@ -497,17 +554,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
554
def __del__(self):
555
self.shutdown()
556
500
-
501
-class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
502
-
503
- def __init__(self, *args, **kwargs):
504
- super().__init__(*args, **kwargs)
505
- self.pp_locks: Optional[List[asyncio.Lock]] = None
506
- self.use_ray_spmd_worker = envs.VLLM_USE_RAY_SPMD_WORKER
507
- if not self.use_ray_compiled_dag:
508
- self.driver_exec_method = make_async(
509
- self.driver_worker.execute_method)
510
-
557
async def execute_model_async(
558
self,
559
execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
@@ -568,5 +614,7 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
614
]
615
return await asyncio.gather(*coros)
616
571
- def __del__(self):
572
- self.shutdown()
617
+ def check_health(self) -> None:
618
+ # Assume that the Ray workers are healthy.
619
+ # TODO: check the health of the Ray workers
620
+ return
vllm/executor/ray_hpu_executor.py
deleted
-515
@@ -1,515 +0,0 @@
1
-import asyncio
2
-import os
3
-from collections import defaultdict
4
-from itertools import islice, repeat
5
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
6
-
7
-import msgspec
8
-
9
-import vllm.envs as envs
10
-from vllm.executor.distributed_gpu_executor import ( # yapf: disable
11
- DistributedGPUExecutor, DistributedGPUExecutorAsync)
12
-from vllm.executor.msgspec_utils import encode_hook
13
-from vllm.executor.ray_utils import RayWorkerWrapper, ray
14
-from vllm.logger import init_logger
15
-from vllm.model_executor.layers.sampler import SamplerOutput
16
-from vllm.sequence import ExecuteModelRequest
17
-from vllm.utils import (_run_task_with_lock, get_distributed_init_method,
18
- get_ip, get_open_port, make_async)
19
-
20
-if ray is not None:
21
- from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
22
-
23
-if TYPE_CHECKING:
24
- from ray.util.placement_group import PlacementGroup
25
-
26
-logger = init_logger(__name__)
27
-
28
-
29
-class RayHPUExecutor(DistributedGPUExecutor):
30
-
31
- uses_ray: bool = True
32
-
33
- def _init_executor(self) -> None:
34
- self.forward_dag: Optional[ray.dag.CompiledDAG] = None
35
- # If the env var is set, it uses the Ray's compiled DAG API
36
- # which optimizes the control plane overhead.
37
- # Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
38
- # Currently, this requires USE_RAY_SPMD_WORKER=True.
39
- self.use_ray_compiled_dag = envs.VLLM_USE_RAY_COMPILED_DAG
40
- # If the env var is set, then we do not distinguish between the
41
- # "driver worker" vs other workers. Also, the rank 0 worker will
42
- # be executed in a remote Ray worker. Currently this requires
43
- # USE_RAY_COMPILED_DAG=True.
44
- self.use_ray_spmd_worker = envs.VLLM_USE_RAY_SPMD_WORKER
45
- if self.use_ray_compiled_dag:
46
- assert self.use_ray_spmd_worker, (
47
- "VLLM_USE_RAY_COMPILED_DAG=1 requires "
48
- "VLLM_USE_RAY_SPMD_WORKER=1")
49
- if self.use_ray_spmd_worker:
50
- # TODO: Support SPMD worker for non-DAG Ray executor.
51
- assert self.use_ray_compiled_dag, (
52
- "VLLM_USE_RAY_SPMD_WORKER=1 requires "
53
- "VLLM_USE_RAY_COMPILED_DAG=1")
54
-
55
- assert self.uses_ray
56
- placement_group = self.parallel_config.placement_group
57
-
58
- # Disable Ray usage stats collection.
59
- ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0")
60
- if ray_usage != "1":
61
- os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
62
-
63
- # Create the parallel GPU workers.
64
- self._init_workers_ray(placement_group)
65
-
66
- self.input_encoder = msgspec.msgpack.Encoder(enc_hook=encode_hook)
67
- self.output_decoder = msgspec.msgpack.Decoder(
68
- Optional[List[SamplerOutput]])
69
-
70
- def shutdown(self) -> None:
71
- if hasattr(self, "forward_dag") and self.forward_dag is not None:
72
- self.forward_dag.teardown()
73
- import ray
74
- for worker in self.workers:
75
- ray.kill(worker)
76
- self.forward_dag = None
77
-
78
- def finish_measurements(self):
79
- self._run_workers("finish_measurements")
80
-
81
- def _init_workers_ray(self, placement_group: "PlacementGroup",
82
- **ray_remote_kwargs):
83
- # Otherwise, the ray workers are allocated with a full GPU.
84
- num_gpus = 1
85
-
86
- # The driver dummy worker does not actually use any resources.
87
- # It holds the resource for the driver worker.
88
- self.driver_dummy_worker: Optional[RayWorkerWrapper] = None
89
- # The remaining workers are the actual ray actors.
90
- self.workers: List[RayWorkerWrapper] = []
91
-
92
- # Used in ray compiled DAG: indexed first by PP rank,
93
- # and then TP rank. In other words, the inner list is
94
- # the TP group of workers for a PP rank.
95
- self.pp_tp_workers: List[List[RayWorkerWrapper]] = []
96
-
97
- logger.info("use_ray_spmd_worker: %s", self.use_ray_spmd_worker)
98
-
99
- # Create the workers.
100
- driver_ip = get_ip()
101
- for bundle_id, bundle in enumerate(placement_group.bundle_specs):
102
- if not bundle.get("HPU", 0):
103
- continue
104
- scheduling_strategy = PlacementGroupSchedulingStrategy(
105
- placement_group=placement_group,
106
- placement_group_capture_child_tasks=True,
107
- placement_group_bundle_index=bundle_id,
108
- )
109
-
110
- worker = ray.remote(
111
- num_cpus=0,
112
- num_gpus=0,
113
- resources={'HPU': num_gpus},
114
- scheduling_strategy=scheduling_strategy,
115
- **ray_remote_kwargs,
116
- )(RayWorkerWrapper).remote(vllm_config=self.vllm_config)
117
-
118
- if self.use_ray_spmd_worker:
119
- self.workers.append(worker)
120
- else:
121
- worker_ip = ray.get(worker.get_node_ip.remote())
122
- if worker_ip == driver_ip and self.driver_dummy_worker is None:
123
- # If the worker is on the same node as the driver, we use it
124
- # as the resource holder for the driver process.
125
- self.driver_dummy_worker = worker
126
- self.driver_worker = RayWorkerWrapper(
127
- vllm_config=self.vllm_config)
128
- else:
129
- # Else, added to the list of workers.
130
- self.workers.append(worker)
131
-
132
- logger.debug("workers: %s", self.workers)
133
- logger.debug("driver_dummy_worker: %s", self.driver_dummy_worker)
134
- if not self.use_ray_spmd_worker and self.driver_dummy_worker is None:
135
- raise ValueError(
136
- "Ray does not allocate any GPUs on the driver node. Consider "
137
- "adjusting the Ray placement group or running the driver on a "
138
- "GPU node.")
139
-
140
- worker_ips = [
141
- ray.get(worker.get_node_ip.remote()) # type: ignore[attr-defined]
142
- for worker in self.workers
143
- ]
144
- ip_counts: Dict[str, int] = {}
145
- for ip in worker_ips:
146
- ip_counts[ip] = ip_counts.get(ip, 0) + 1
147
-
148
- def sort_by_driver_then_worker_ip(worker):
149
- """
150
- Sort the workers based on 3 properties:
151
- 1. If the worker is on the same node as the driver (vllm engine),
152
- it should be placed first.
153
- 2. Then, if the worker is on a node with fewer workers, it should
154
- be placed first.
155
- 3. Finally, if the work is on a node with smaller IP address, it
156
- should be placed first.
157
- """
158
- ip = ray.get(worker.get_node_ip.remote())
159
- return (ip != driver_ip, ip_counts[ip], ip)
160
-
161
- # After sorting, the workers on the same node will be
162
- # close to each other, and the workers on the driver
163
- # node will be placed first.
164
- self.workers = sorted(self.workers, key=sort_by_driver_then_worker_ip)
165
-
166
- worker_node_and_gpu_ids = []
167
- for worker in [self.driver_dummy_worker] + self.workers:
168
- if worker is None:
169
- # driver_dummy_worker can be None when using ray spmd worker.
170
- continue
171
- worker_node_and_gpu_ids.append(
172
- ray.get(worker.get_node_and_gpu_ids.remote()) \
173
- ) # type: ignore
174
-
175
- node_workers = defaultdict(list) # node id -> list of worker ranks
176
- node_gpus = defaultdict(list) # node id -> list of gpu ids
177
-
178
- for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
179
- node_workers[node_id].append(i)
180
- # `gpu_ids` can be a list of strings or integers.
181
- # convert them to integers for consistency.
182
- # NOTE: gpu_ids can be larger than 9 (e.g. 16 GPUs),
183
- # string sorting is not sufficient.
184
- # see https://github.com/vllm-project/vllm/issues/5590
185
- gpu_ids = [int(x) for x in gpu_ids]
186
- node_gpus[node_id].extend(gpu_ids)
187
- for node_id, gpu_ids in node_gpus.items():
188
- node_gpus[node_id] = sorted(gpu_ids)
189
-
190
- all_ips = set(worker_ips + [driver_ip])
191
- n_ips = len(all_ips)
192
- n_nodes = len(node_workers)
193
-
194
- if n_nodes != n_ips:
195
- raise RuntimeError(
196
- f"Every node should have a unique IP address. Got {n_nodes}"
197
- f" nodes with node ids {list(node_workers.keys())} and "
198
- f"{n_ips} unique IP addresses {all_ips}. Please check your"
199
- " network configuration. If you set `VLLM_HOST_IP` "
200
- "environment variable, make sure it is unique for"
201
- " each node.")
202
-
203
- # Set environment variables for the driver and workers.
204
- all_args_to_update_environment_variables = [({
205
- "VLLM_TRACE_FUNCTION":
206
- str(envs.VLLM_TRACE_FUNCTION),
207
- }, ) for (node_id, _) in worker_node_and_gpu_ids]
208
- self._run_workers("update_environment_variables",
209
- all_args=all_args_to_update_environment_variables)
210
-
211
- if len(node_gpus) == 1:
212
- # in single node case, we don't need to get the IP address.
213
- # the loopback address is sufficient
214
- # NOTE: a node may have several IP addresses, one for each
215
- # network interface. `get_ip()` might return any of them,
216
- # while they might not work for communication inside the node
217
- # if the network setup is complicated. Using the loopback address
218
- # solves this issue, as it always works for communication inside
219
- # the node.
220
- driver_ip = "127.0.0.1"
221
- distributed_init_method = get_distributed_init_method(
222
- driver_ip, get_open_port())
223
-
224
- # Initialize the actual workers inside worker wrapper.
225
- init_worker_all_kwargs = [
226
- self._get_worker_kwargs(
227
- local_rank=node_workers[node_id].index(rank),
228
- rank=rank,
229
- distributed_init_method=distributed_init_method,
230
- ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids)
231
- ]
232
- self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
233
-
234
- self._run_workers("init_device")
235
- self._run_workers("load_model",
236
- max_concurrent_workers=self.parallel_config.
237
- max_parallel_loading_workers)
238
-
239
- if self.use_ray_spmd_worker:
240
- for pp_rank in range(self.parallel_config.pipeline_parallel_size):
241
- self.pp_tp_workers.append([])
242
- for tp_rank in range(
243
- self.parallel_config.tensor_parallel_size):
244
- # PP=2, TP=4
245
- # pp_tp_workers = [[0, 1, 2, 3], [4, 5, 6, 7]]
246
- rank = (pp_rank * self.parallel_config.tensor_parallel_size
247
- ) + tp_rank
248
- assert len(self.pp_tp_workers[pp_rank]) == tp_rank
249
- assert pp_rank < len(self.pp_tp_workers)
250
- self.pp_tp_workers[pp_rank].append(self.workers[rank])
251
-
252
- # This is the list of workers that are rank 0 of each TP group EXCEPT
253
- # global rank 0. These are the workers that will broadcast to the
254
- # rest of the workers.
255
- self.tp_driver_workers: List[RayWorkerWrapper] = []
256
- # This is the list of workers that are not drivers and not the first
257
- # worker in a TP group. These are the workers that will be
258
- # broadcasted to.
259
- self.non_driver_workers: List[RayWorkerWrapper] = []
260
-
261
- # Enforce rank order for correct rank to return final output.
262
- for index, worker in enumerate(self.workers):
263
- # The driver worker is rank 0 and not in self.workers.
264
- rank = index + 1
265
- if rank % self.parallel_config.tensor_parallel_size == 0:
266
- self.tp_driver_workers.append(worker)
267
- else:
268
- self.non_driver_workers.append(worker)
269
-
270
- def _driver_execute_model(
271
- self, execute_model_req: Optional[ExecuteModelRequest]
272
- ) -> Optional[List[SamplerOutput]]:
273
- """Run execute_model in the driver worker.
274
-
275
- Passing None will cause the driver to stop the model execution
276
- loop running in each of the remote workers.
277
- """
278
- assert not self.use_ray_spmd_worker, (
279
- "driver_worker does not exist for VLLM_USE_RAY_SPMD_WORKER=1")
280
- return self.driver_worker.execute_method("execute_model",
281
- execute_model_req)
282
-
283
- def execute_model(
284
- self,
285
- execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
286
- if not self.use_ray_spmd_worker:
287
- return super().execute_model(execute_model_req)
288
-
289
- if self.forward_dag is None:
290
- self.forward_dag = self._compiled_ray_dag(enable_asyncio=False)
291
-
292
- serialized_data = self.input_encoder.encode(execute_model_req)
293
- outputs = ray.get(self.forward_dag.execute(serialized_data))
294
- output = self.output_decoder.decode(outputs[0])
295
- return output
296
-
297
- def _run_workers(
298
- self,
299
- method: str,
300
- *args,
301
- async_run_tensor_parallel_workers_only: bool = False,
302
- all_args: Optional[List[Tuple[Any, ...]]] = None,
303
- all_kwargs: Optional[List[Dict[str, Any]]] = None,
304
- max_concurrent_workers: Optional[int] = None,
305
- **kwargs,
306
- ) -> Any:
307
- """Runs the given method on all workers. Can be used in the following
308
- ways:
309
-
310
- Args:
311
- - async_run_tensor_parallel_workers_only: If True the method will be
312
- run only in the remote TP workers, not the driver worker.
313
- It will also be run asynchronously and return a list of futures
314
- rather than blocking on the results.
315
- - args/kwargs: All workers share the same args/kwargs
316
- - all_args/all_kwargs: args/kwargs for each worker are specified
317
- individually
318
- """
319
- if self.use_ray_spmd_worker:
320
- assert not async_run_tensor_parallel_workers_only, (
321
- "async_run_tensor_parallel_workers_only is not supported for "
322
- "spmd mode.")
323
-
324
- if max_concurrent_workers:
325
- raise NotImplementedError(
326
- "max_concurrent_workers is not supported yet.")
327
-
328
- count = len(self.workers) if not \
329
- async_run_tensor_parallel_workers_only \
330
- else len(self.non_driver_workers)
331
- # If using SPMD worker, all workers are the same, so we should execute
332
- # the args on all workers. Otherwise, we skip the first worker's args
333
- # because those args will go to the driver worker.
334
- first_worker_args_index: int = 0 if self.use_ray_spmd_worker else 1
335
- all_worker_args = repeat(args, count) if all_args is None \
336
- else islice(all_args, first_worker_args_index, None)
337
- all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
338
- else islice(all_kwargs, first_worker_args_index, None)
339
-
340
- # Start the ray workers first.
341
- ray_workers = self.workers
342
- if async_run_tensor_parallel_workers_only:
343
- ray_workers = self.non_driver_workers
344
- ray_worker_outputs = [
345
- worker.execute_method.remote(method, *worker_args, **worker_kwargs)
346
- for (worker, worker_args, worker_kwargs
347
- ) in zip(ray_workers, all_worker_args, all_worker_kwargs)
348
- ]
349
-
350
- if async_run_tensor_parallel_workers_only:
351
- # Just return futures
352
- return ray_worker_outputs
353
-
354
- driver_worker_output = []
355
- # In SPMD mode, the driver worker is the same as any other worker,
356
- # so we only explicitly execute on the driver worker if using a
357
- # non-SPMD worker class.
358
- if not self.use_ray_spmd_worker:
359
- driver_args = args if all_args is None else all_args[0]
360
- driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
361
-
362
- # Start the driver worker after all the ray workers.
363
- driver_worker_output = [
364
- self.driver_worker.execute_method(method, *driver_args,
365
- **driver_kwargs)
366
- ]
367
-
368
- # Get the results of the ray workers.
369
- if self.workers:
370
- ray_worker_outputs = ray.get(ray_worker_outputs)
371
-
372
- return driver_worker_output + ray_worker_outputs
373
-
374
- def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
375
- """Wait for futures returned from _run_workers() with
376
- async_run_remote_workers_only to complete."""
377
- ray.get(parallel_worker_tasks)
378
-
379
- def _check_ray_adag_installation(self):
380
- import pkg_resources
381
- from packaging import version
382
-
383
- required_version = version.parse("2.35")
384
- current_version = version.parse(
385
- pkg_resources.get_distribution("ray").version)
386
- # TODO: update the constraint once we adapt to the backward
387
- # incompatible API change from ray 2.36
388
- if current_version != required_version:
389
- raise ValueError(f"Ray version {required_version} is "
390
- f"required, but found {current_version}")
391
-
392
- import importlib.util
393
- adag_spec = importlib.util.find_spec(
394
- "ray.experimental.compiled_dag_ref")
395
- if adag_spec is None:
396
- raise ValueError("Ray accelerated DAG is not installed. "
397
- "Run `pip install ray[adag]` to install it.")
398
-
399
- def _compiled_ray_dag(self, enable_asyncio: bool):
400
- assert self.parallel_config.use_ray
401
- self._check_ray_adag_installation()
402
- from ray.dag import InputNode, MultiOutputNode
403
- from ray.experimental.channel.torch_tensor_type import TorchTensorType
404
-
405
- with InputNode() as input_data:
406
- # Example DAG: PP=2, TP=4
407
- # (ExecuteModelReq, None) -> 0 -> (ExecuteModelReq, IntermediateOutput) -> 4 -> SamplerOutput # noqa: E501
408
- # -> 1 -> (ExecuteModelReq, IntermediateOutput) -> 5 -> SamplerOutput # noqa: E501
409
- # -> 2 -> (ExecuteModelReq, IntermediateOutput) -> 6 -> SamplerOutput # noqa: E501
410
- # -> 3 -> (ExecuteModelReq, IntermediateOutput) -> 7 -> SamplerOutput # noqa: E501
411
-
412
- # All workers in the first TP group will take in the
413
- # ExecuteModelRequest as input.
414
- outputs = [input_data for _ in self.pp_tp_workers[0]]
415
- for pp_rank, tp_group in enumerate(self.pp_tp_workers):
416
- # Each PP worker takes in the output of the previous PP worker,
417
- # and the TP group executes in SPMD fashion.
418
- outputs = [
419
- worker.execute_model_spmd.
420
- bind( # type: ignore[attr-defined]
421
- outputs[i]) for i, worker in enumerate(tp_group)
422
- ]
423
-
424
- last_pp_rank = len(self.pp_tp_workers) - 1
425
- if pp_rank < last_pp_rank:
426
- # Specify how intermediate tensors should be passed
427
- # between pp stages, no need to specify for the last
428
- # pp stage.
429
- transport = "auto"
430
- outputs = [
431
- output.with_type_hint(
432
- TorchTensorType(transport=transport))
433
- for output in outputs
434
- ]
435
-
436
- forward_dag = MultiOutputNode(outputs)
437
-
438
- return forward_dag.experimental_compile(enable_asyncio=enable_asyncio)
439
-
440
- def __del__(self):
441
- self.shutdown()
442
-
443
-
444
-class RayHPUExecutorAsync(RayHPUExecutor, DistributedGPUExecutorAsync):
445
-
446
- def __init__(self, *args, **kwargs):
447
- super().__init__(*args, **kwargs)
448
- self.pp_locks: Optional[List[asyncio.Lock]] = None
449
- self.use_ray_spmd_worker = envs.VLLM_USE_RAY_SPMD_WORKER
450
- if not self.use_ray_compiled_dag:
451
- self.driver_exec_method = make_async(
452
- self.driver_worker.execute_method)
453
-
454
- async def execute_model_async(
455
- self,
456
- execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
457
- if not self.use_ray_spmd_worker:
458
- return await super().execute_model_async(execute_model_req)
459
-
460
- if self.forward_dag is None:
461
- self.forward_dag = self._compiled_ray_dag(enable_asyncio=True)
462
-
463
- serialized_data = self.input_encoder.encode(execute_model_req)
464
- dag_future = await self.forward_dag.execute_async(serialized_data)
465
- outputs = await dag_future
466
- return self.output_decoder.decode(outputs[0])
467
-
468
- async def _driver_execute_model_async(
469
- self,
470
- execute_model_req: Optional[ExecuteModelRequest] = None
471
- ) -> List[SamplerOutput]:
472
- assert not self.use_ray_spmd_worker, (
473
- "driver_worker does not exist for VLLM_USE_RAY_SPMD_WORKER=1")
474
- if not self.tp_driver_workers:
475
- return await self.driver_exec_method("execute_model",
476
- execute_model_req)
477
- if self.pp_locks is None:
478
- # This locks each pipeline parallel stage so multiple virtual
479
- # engines can't execute on the same stage at the same time
480
- # We create the locks here to avoid creating them in the constructor
481
- # which uses a different asyncio loop.
482
- self.pp_locks = [
483
- asyncio.Lock()
484
- for _ in range(self.parallel_config.pipeline_parallel_size)
485
- ]
486
-
487
- tasks = [
488
- asyncio.create_task(
489
- _run_task_with_lock(self.driver_exec_method, self.pp_locks[0],
490
- "execute_model", execute_model_req))
491
- ]
492
- for pp_rank, driver_worker in enumerate(self.tp_driver_workers,
493
- start=1):
494
- tasks.append(
495
- asyncio.create_task(
496
- _run_task_with_lock(driver_worker.execute_method.remote,
497
- self.pp_locks[pp_rank],
498
- "execute_model", execute_model_req)))
499
-
500
- results = await asyncio.gather(*tasks)
501
-
502
- # Only the last PP stage has the final results.
503
- return results[-1]
504
-
505
- async def _start_worker_execution_loop(self):
506
- assert not self.use_ray_spmd_worker, (
507
- "worker loop is disabled for VLLM_USE_RAY_SPMD_WORKER=1")
508
- coros = [
509
- worker.execute_method.remote("start_worker_execution_loop")
510
- for worker in self.non_driver_workers
511
- ]
512
- return await asyncio.gather(*coros)
513
-
514
- def __del__(self):
515
- self.shutdown()
vllm/executor/ray_tpu_executor.py
deleted
-343
@@ -1,343 +0,0 @@
1
-import asyncio
2
-import os
3
-from collections import defaultdict
4
-from itertools import islice, repeat
5
-from typing import (TYPE_CHECKING, Any, Awaitable, Dict, List, Optional, Tuple,
6
- Union)
7
-
8
-import vllm.envs as envs
9
-from vllm.executor.executor_base import ExecutorAsyncBase
10
-from vllm.executor.ray_utils import RayWorkerWrapper, ray
11
-from vllm.executor.tpu_executor import TPUExecutor
12
-from vllm.logger import init_logger
13
-from vllm.model_executor.layers.sampler import SamplerOutput
14
-from vllm.sequence import ExecuteModelRequest
15
-from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
16
- make_async)
17
-
18
-if ray is not None:
19
- from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
20
-
21
-if TYPE_CHECKING:
22
- from ray.util.placement_group import PlacementGroup
23
-
24
-logger = init_logger(__name__)
25
-
26
-
27
-class RayTPUExecutor(TPUExecutor):
28
-
29
- uses_ray: bool = True
30
-
31
- def __init__(self, *args, **kwargs):
32
- # This is non-None when the execute model loop is running
33
- # in the parallel workers. It's a coroutine in the AsyncLLMEngine case.
34
- self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None
35
- # Updated by implementations that require additional args to be passed
36
- # to the _run_workers execute_model call
37
- self.extra_execute_model_run_workers_kwargs: Dict[str, Any] = {}
38
-
39
- super().__init__(*args, **kwargs)
40
-
41
- def _init_executor(self) -> None:
42
- assert self.parallel_config.distributed_executor_backend == "ray"
43
- placement_group = self.parallel_config.placement_group
44
-
45
- # Disable Ray usage stats collection.
46
- ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0")
47
- if ray_usage != "1":
48
- os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
49
-
50
- # Create the parallel TPU workers.
51
- self._init_workers_ray(placement_group)
52
-
53
- def _init_workers_ray(self, placement_group: "PlacementGroup",
54
- **ray_remote_kwargs):
55
- # The driver dummy worker does not actually use any resources.
56
- # It holds the resource for the driver worker.
57
- self.driver_dummy_worker: Optional[RayWorkerWrapper] = None
58
- # The remaining workers are the actual ray actors.
59
- self.workers: List[RayWorkerWrapper] = []
60
-
61
- # Create the workers.
62
- driver_ip = get_ip()
63
- for bundle_id, bundle in enumerate(placement_group.bundle_specs):
64
- if not bundle.get("TPU", 0):
65
- continue
66
- scheduling_strategy = PlacementGroupSchedulingStrategy(
67
- placement_group=placement_group,
68
- placement_group_capture_child_tasks=True,
69
- placement_group_bundle_index=bundle_id,
70
- )
71
-
72
- # GKE does not fetch environment information from metadata server
73
- # and instead sets these from within the Ray process. Therefore we
74
- # need to override the Ray environment variables manually.
75
- override_env = {}
76
- if "TPU_CHIPS_PER_HOST_BOUNDS" in os.environ:
77
- override_env.update({
78
- "TPU_CHIPS_PER_HOST_BOUNDS":
79
- os.environ["TPU_CHIPS_PER_HOST_BOUNDS"]
80
- })
81
- if "TPU_HOST_BOUNDS" in os.environ:
82
- override_env.update(
83
- {"TPU_HOST_BOUNDS": os.environ["TPU_HOST_BOUNDS"]})
84
-
85
- worker = ray.remote(
86
- num_cpus=0,
87
- resources={"TPU": 1},
88
- scheduling_strategy=scheduling_strategy,
89
- **ray_remote_kwargs,
90
- )(RayWorkerWrapper).remote(vllm_config=self.vllm_config)
91
- if override_env:
92
- worker.override_env_vars.remote(override_env)
93
-
94
- worker_ip = ray.get(worker.get_node_ip.remote())
95
- if worker_ip == driver_ip and self.driver_dummy_worker is None:
96
- # If the worker is on the same node as the driver, we use it
97
- # as the resource holder for the driver process.
98
- self.driver_dummy_worker = worker
99
- self.driver_worker = RayWorkerWrapper(
100
- vllm_config=self.vllm_config)
101
- else:
102
- # Else, added to the list of workers.
103
- self.workers.append(worker)
104
-
105
- logger.debug("workers: %s", self.workers)
106
- logger.debug("driver_dummy_worker: %s", self.driver_dummy_worker)
107
- if self.driver_dummy_worker is None:
108
- raise ValueError(
109
- "Ray does not allocate any TPUs on the driver node. Consider "
110
- "adjusting the Ray placement group or running the driver on a "
111
- "TPU node.")
112
-
113
- worker_ips = [
114
- ray.get(worker.get_node_ip.remote()) # type: ignore[attr-defined]
115
- for worker in self.workers
116
- ]
117
- ip_counts: Dict[str, int] = {}
118
- for ip in worker_ips:
119
- ip_counts[ip] = ip_counts.get(ip, 0) + 1
120
-
121
- def sort_by_driver_then_worker_ip(worker):
122
- """
123
- Sort the workers based on 3 properties:
124
- 1. If the worker is on the same node as the driver (vllm engine),
125
- it should be placed first.
126
- 2. Then, if the worker is on a node with fewer workers, it should
127
- be placed first.
128
- 3. Finally, if the work is on a node with smaller IP address, it
129
- should be placed first.
130
- """
131
- ip = ray.get(worker.get_node_ip.remote())
132
- return (ip != driver_ip, ip_counts[ip], ip)
133
-
134
- # After sorting, the workers on the same node will be
135
- # close to each other, and the workers on the driver
136
- # node will be placed first.
137
- self.workers = sorted(self.workers, key=sort_by_driver_then_worker_ip)
138
-
139
- # Get the set of TPU IDs used on each node.
140
- worker_node_and_gpu_ids = []
141
- for worker in [self.driver_dummy_worker] + self.workers:
142
- if worker is None:
143
- # driver_dummy_worker can be None when using ray spmd worker.
144
- continue
145
- worker_node_and_gpu_ids.append(
146
- ray.get(worker.get_node_and_gpu_ids.remote()) \
147
- ) # type: ignore
148
-
149
- node_workers = defaultdict(list)
150
- for i, (node_id, _) in enumerate(worker_node_and_gpu_ids):
151
- node_workers[node_id].append(i)
152
-
153
- # Set environment variables for the driver and workers.
154
- all_args_to_update_environment_variables = [({
155
- "VLLM_TRACE_FUNCTION":
156
- str(envs.VLLM_TRACE_FUNCTION),
157
- }, ) for _ in worker_node_and_gpu_ids]
158
- self._run_workers("update_environment_variables",
159
- all_args=all_args_to_update_environment_variables)
160
-
161
- if len(node_workers) == 1:
162
- # in single node case, we don't need to get the IP address.
163
- # the loopback address is sufficient
164
- # NOTE: a node may have several IP addresses, one for each
165
- # network interface. `get_ip()` might return any of them,
166
- # while they might not work for communication inside the node
167
- # if the network setup is complicated. Using the loopback address
168
- # solves this issue, as it always works for communication inside
169
- # the node.
170
- driver_ip = "127.0.0.1"
171
- distributed_init_method = get_distributed_init_method(
172
- driver_ip, get_open_port())
173
-
174
- # Initialize the actual workers inside worker wrapper.
175
- init_worker_all_kwargs = [
176
- self._get_worker_kwargs(
177
- local_rank=node_workers[node_id].index(rank),
178
- rank=rank,
179
- distributed_init_method=distributed_init_method,
180
- ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids)
181
- ]
182
- self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
183
-
184
- self._run_workers("init_device")
185
- self._run_workers("load_model",
186
- max_concurrent_workers=self.parallel_config.
187
- max_parallel_loading_workers)
188
-
189
- def _driver_execute_model(
190
- self,
191
- execute_model_req: Optional[ExecuteModelRequest] = None
192
- ) -> List[SamplerOutput]:
193
- """Run execute_model in the driver worker.
194
-
195
- Passing None will cause the driver to stop the model execution
196
- loop running in each of the remote workers.
197
- """
198
- return self.driver_worker.execute_method("execute_model",
199
- execute_model_req)
200
-
201
- def _run_workers(
202
- self,
203
- method: str,
204
- *args,
205
- async_run_remote_workers_only: bool = False,
206
- all_args: Optional[List[Tuple[Any, ...]]] = None,
207
- all_kwargs: Optional[List[Dict[str, Any]]] = None,
208
- max_concurrent_workers: Optional[int] = None,
209
- use_ray_compiled_dag: bool = False,
210
- **kwargs,
211
- ) -> Any:
212
- """Runs the given method on all workers. Can be used in the following
213
- ways:
214
-
215
- - async_run_remote_workers_only: If True the method will be run only
216
- in the remote workers, not the driver worker. It will also be
217
- run asynchronously and return a list of futures rather than blocking
218
- on the results.
219
- - args/kwargs: All workers share the same args/kwargs
220
- - all_args/all_kwargs: args/kwargs for each worker are specified
221
- individually
222
- """
223
-
224
- if max_concurrent_workers:
225
- raise NotImplementedError(
226
- "max_concurrent_workers is not supported yet.")
227
-
228
- count = len(self.workers)
229
- all_worker_args = repeat(args, count) if all_args is None \
230
- else islice(all_args, 1, None)
231
- all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
232
- else islice(all_kwargs, 1, None)
233
-
234
- # Start the ray workers first.
235
- ray_worker_outputs = [
236
- worker.execute_method.remote(method, *worker_args, **worker_kwargs)
237
- for (worker, worker_args, worker_kwargs
238
- ) in zip(self.workers, all_worker_args, all_worker_kwargs)
239
- ]
240
-
241
- if async_run_remote_workers_only:
242
- # Just return futures
243
- return ray_worker_outputs
244
-
245
- driver_args = args if all_args is None else all_args[0]
246
- driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
247
-
248
- # Start the driver worker after all the ray workers.
249
- driver_worker_output = self.driver_worker.execute_method(
250
- method, *driver_args, **driver_kwargs)
251
- # Get the results of the ray workers.
252
- if self.workers:
253
- ray_worker_outputs = ray.get(ray_worker_outputs)
254
-
255
- return [driver_worker_output] + ray_worker_outputs
256
-
257
- def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
258
- """Wait for futures returned from _run_workers() with
259
- async_run_remote_workers_only to complete."""
260
- ray.get(parallel_worker_tasks)
261
-
262
- def determine_num_available_blocks(self) -> Tuple[int, int]:
263
- num_blocks = self._run_workers("determine_num_available_blocks", )
264
- num_tpu_blocks = min(b[0] for b in num_blocks)
265
- num_cpu_blocks = min(b[1] for b in num_blocks)
266
- return num_tpu_blocks, num_cpu_blocks
267
-
268
- def initialize_cache(self, num_gpu_blocks: int,
269
- num_cpu_blocks: int) -> None:
270
- logger.info("# TPU blocks: %d, # CPU blocks: %d", num_gpu_blocks,
271
- num_cpu_blocks)
272
- self.cache_config.num_gpu_blocks = num_gpu_blocks
273
- self.cache_config.num_cpu_blocks = num_cpu_blocks
274
- self._run_workers("initialize_cache",
275
- num_gpu_blocks=num_gpu_blocks,
276
- num_cpu_blocks=num_cpu_blocks)
277
-
278
- def execute_model(
279
- self,
280
- execute_model_req: ExecuteModelRequest,
281
- ) -> List[SamplerOutput]:
282
- if self.parallel_worker_tasks is None:
283
- self.parallel_worker_tasks = self._run_workers(
284
- "start_worker_execution_loop",
285
- async_run_remote_workers_only=True,
286
- **self.extra_execute_model_run_workers_kwargs)
287
-
288
- # Only the driver worker returns the sampling results.
289
- return self._driver_execute_model(execute_model_req)
290
-
291
- def stop_remote_worker_execution_loop(self) -> None:
292
- if self.parallel_worker_tasks is None:
293
- return
294
-
295
- self._driver_execute_model()
296
- parallel_worker_tasks = self.parallel_worker_tasks
297
- self.parallel_worker_tasks = None
298
- # Ensure that workers exit model loop cleanly
299
- # (this will raise otherwise)
300
- self._wait_for_tasks_completion(parallel_worker_tasks)
301
-
302
-
303
-class RayTPUExecutorAsync(RayTPUExecutor, ExecutorAsyncBase):
304
-
305
- def __init__(self, *args, **kwargs):
306
- super().__init__(*args, **kwargs)
307
- self.driver_exec_method = make_async(self.driver_worker.execute_method)
308
-
309
- async def execute_model_async(
310
- self,
311
- execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
312
- if self.parallel_worker_tasks is None:
313
- # Start model execution loop running in the parallel workers
314
- self.parallel_worker_tasks = asyncio.create_task(
315
- self._start_worker_execution_loop())
316
-
317
- # Only the driver worker returns the sampling results.
318
- return await self._driver_execute_model_async(execute_model_req)
319
-
320
- async def stop_remote_worker_execution_loop_async(self) -> None:
321
- if self.parallel_worker_tasks is None:
322
- return
323
-
324
- await self._driver_execute_model_async()
325
- parallel_worker_tasks = self.parallel_worker_tasks
326
- self.parallel_worker_tasks = None
327
- # Ensure that workers exit model loop cleanly
328
- # (this will raise otherwise)
329
- await parallel_worker_tasks
330
-
331
- async def _driver_execute_model_async(
332
- self,
333
- execute_model_req: Optional[ExecuteModelRequest] = None
334
- ) -> List[SamplerOutput]:
335
- return await self.driver_exec_method("execute_model",
336
- execute_model_req)
337
-
338
- async def _start_worker_execution_loop(self):
339
- coros = [
340
- worker.execute_method.remote("start_worker_execution_loop")
341
- for worker in self.workers
342
- ]
343
- return await asyncio.gather(*coros)
vllm/executor/ray_utils.py
+25
-1
@@ -1,7 +1,7 @@
1
import os
2
import time
3
from collections import defaultdict
4
-from typing import Dict, List, Optional, Tuple, Union
4
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
5
6
import msgspec
7
@@ -13,6 +13,10 @@ from vllm.sequence import ExecuteModelRequest, IntermediateTensors
13
from vllm.utils import get_ip
14
from vllm.worker.worker_base import WorkerWrapperBase
15
16
+if TYPE_CHECKING:
17
+ from vllm.v1.core.scheduler import SchedulerOutput
18
+ from vllm.v1.outputs import ModelRunnerOutput
19
+
20
logger = init_logger(__name__)
21
PG_WAIT_TIMEOUT = 1800
22
@@ -95,6 +99,26 @@ try:
99
100
return output
101
102
+ def setup_device_if_necessary(self):
103
+ # TODO(swang): This is needed right now because Ray CG executes
104
+ # on a background thread, so we need to reset torch's current
105
+ # device.
106
+ # We can remove this API after it is fixed in compiled graph.
107
+ import torch
108
+ assert self.worker is not None, "Worker is not initialized"
109
+ if not self.compiled_dag_cuda_device_set:
110
+ torch.cuda.set_device(self.worker.device)
111
+ self.compiled_dag_cuda_device_set = True
112
+
113
+ def execute_model(
114
+ self,
115
+ scheduler_output: "SchedulerOutput",
116
+ ) -> "ModelRunnerOutput":
117
+ self.setup_device_if_necessary()
118
+ assert self.worker is not None, "Worker is not initialized"
119
+ output = self.worker.model_runner.execute_model(scheduler_output)
120
+ return output
121
+
122
def override_env_vars(self, vars: Dict[str, str]):
123
os.environ.update(vars)
124
vllm/executor/ray_xpu_executor.py
deleted
-40
@@ -1,40 +0,0 @@
1
-import asyncio
2
-from typing import List, Optional
3
-
4
-import ray
5
-
6
-import vllm.envs as envs
7
-from vllm.executor.ray_gpu_executor import RayGPUExecutor, RayGPUExecutorAsync
8
-from vllm.executor.xpu_executor import XPUExecutor
9
-from vllm.logger import init_logger
10
-from vllm.utils import make_async
11
-
12
-logger = init_logger(__name__)
13
-
14
-
15
-class RayXPUExecutor(RayGPUExecutor, XPUExecutor):
16
-
17
- def _get_env_vars_to_be_updated(self):
18
- # Get the set of GPU IDs used on each node.
19
- worker_node_and_gpu_ids = []
20
- for worker in [self.driver_dummy_worker] + self.workers:
21
- if worker is None:
22
- # driver_dummy_worker can be None when using ray spmd worker.
23
- continue
24
- worker_node_and_gpu_ids.append(
25
- ray.get(worker.get_node_and_gpu_ids.remote())) # type: ignore
26
-
27
- # Set environment variables for the driver and workers.
28
- all_args_to_update_environment_variables = [({
29
- "VLLM_TRACE_FUNCTION":
30
- str(envs.VLLM_TRACE_FUNCTION),
31
- }, ) for (_, _) in worker_node_and_gpu_ids]
32
- return all_args_to_update_environment_variables
33
-
34
-
35
-class RayXPUExecutorAsync(RayXPUExecutor, RayGPUExecutorAsync):
36
-
37
- def __init__(self, *args, **kwargs):
38
- super().__init__(*args, **kwargs)
39
- self.driver_exec_method = make_async(self.driver_worker.execute_method)
40
- self.pp_locks: Optional[List[asyncio.Lock]] = None
vllm/executor/tpu_executor.py
deleted
-142
@@ -1,142 +0,0 @@
1
-from typing import Any, Dict, List, Optional, Set, Tuple
2
-
3
-import torch
4
-
5
-from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
6
-from vllm.logger import init_logger
7
-from vllm.lora.request import LoRARequest
8
-from vllm.model_executor.layers.sampler import SamplerOutput
9
-from vllm.sequence import ExecuteModelRequest
10
-from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
11
- make_async)
12
-
13
-logger = init_logger(__name__)
14
-
15
-
16
-class TPUExecutor(ExecutorBase):
17
-
18
- uses_ray: bool = False
19
-
20
- def _init_executor(self) -> None:
21
- assert not self.scheduler_config.chunked_prefill_enabled, (
22
- "Chunked prefill is not yet supported for TPU backend")
23
- assert not self.speculative_config, (
24
- "Speculative decoding is not yet supported for TPU backend")
25
- if self.model_config.dtype in (torch.float16, torch.float32):
26
- logger.warning(
27
- "The TPU backend currently does not support %s. "
28
- "Using bfloat16 instead.", self.model_config.dtype)
29
- self.model_config.dtype = torch.bfloat16
30
-
31
- # Instantiate the worker and load the model to the device.
32
- self.driver_worker = self._create_worker()
33
- self.driver_worker.init_device()
34
- self.driver_worker.load_model()
35
-
36
- def _get_worker_kwargs(
37
- self,
38
- local_rank: int = 0,
39
- rank: int = 0,
40
- distributed_init_method: Optional[str] = None,
41
- ) -> Dict[str, Any]:
42
- """Return worker init args for a given rank."""
43
- if distributed_init_method is None:
44
- distributed_init_method = get_distributed_init_method(
45
- get_ip(), get_open_port())
46
- return dict(
47
- vllm_config=self.vllm_config,
48
- local_rank=local_rank,
49
- rank=rank,
50
- distributed_init_method=distributed_init_method,
51
- is_driver_worker=rank == 0,
52
- )
53
-
54
- def _create_worker(
55
- self,
56
- local_rank: int = 0,
57
- rank: int = 0,
58
- distributed_init_method: Optional[str] = None,
59
- ):
60
- if self.scheduler_config.is_multi_step:
61
- from vllm.worker.multi_step_tpu_worker import MultiStepTPUWorker
62
- worker = MultiStepTPUWorker(**self._get_worker_kwargs(
63
- local_rank, rank, distributed_init_method))
64
- return worker
65
- else:
66
- from vllm.worker.tpu_worker import TPUWorker
67
-
68
- worker = TPUWorker(**self._get_worker_kwargs(
69
- local_rank, rank, distributed_init_method))
70
- return worker
71
-
72
- def initialize_cache(
73
- self,
74
- num_gpu_blocks: int,
75
- num_cpu_blocks: int,
76
- ) -> None:
77
- """Initialize the KV cache by invoking the underlying worker."""
78
- # NOTE: This is logged in the executor because there can be >1 worker
79
- # with other executors. We could log in the engine level, but work
80
- # remains to abstract away the device for non-GPU configurations.
81
- logger.info("# TPU blocks: %d, # CPU blocks: %d", num_gpu_blocks,
82
- num_cpu_blocks)
83
- self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
84
-
85
- def determine_num_available_blocks(self) -> Tuple[int, int]:
86
- """Determine the number of available KV blocks by invoking the
87
- underlying worker."""
88
- return self.driver_worker.determine_num_available_blocks()
89
-
90
- def execute_model(
91
- self,
92
- execute_model_req: ExecuteModelRequest,
93
- ) -> List[SamplerOutput]:
94
- output = self.driver_worker.execute_model(execute_model_req)
95
- return output
96
-
97
- def add_lora(self, lora_request: LoRARequest) -> bool:
98
- raise NotImplementedError(
99
- "LoRA is currently not supported by the TPU backend.")
100
-
101
- def remove_lora(self, lora_id: int) -> bool:
102
- raise NotImplementedError(
103
- "LoRA is currently not supported by the TPU backend.")
104
-
105
- def pin_lora(self, lora_id: int) -> bool:
106
- raise NotImplementedError(
107
- "LoRA is currently not supported by the TPU backend.")
108
-
109
- def list_loras(self) -> Set[int]:
110
- raise NotImplementedError(
111
- "LoRA is currently not supported by the TPU backend.")
112
-
113
- def add_prompt_adapter(self, prompt_adapter_request) -> bool:
114
- raise NotImplementedError(
115
- "Soft prompt is currently not supported by the TPU backend.")
116
-
117
- def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
118
- raise NotImplementedError(
119
- "Soft prompt is currently not supported by the TPU backend.")
120
-
121
- def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
122
- raise NotImplementedError(
123
- "Soft prompt is currently not supported by the TPU backend.")
124
-
125
- def list_prompt_adapters(self) -> Set[int]:
126
- raise NotImplementedError(
127
- "Soft prompt is currently not supported by the TPU backend.")
128
-
129
- def check_health(self) -> None:
130
- # TPUExecutor will always be healthy as long as it's running.
131
- return
132
-
133
-
134
-class TPUExecutorAsync(TPUExecutor, ExecutorAsyncBase):
135
-
136
- async def execute_model_async(
137
- self,
138
- sexecute_model_req: ExecuteModelRequest,
139
- ) -> SamplerOutput:
140
- output = await make_async(self.driver_worker.execute_model
141
- )(sexecute_model_req)
142
- return output
vllm/executor/uniproc_executor.py
new
+57
@@ -0,0 +1,57 @@
1
+from typing import Any, Dict, List, Optional, Tuple
2
+
3
+from vllm.executor.executor_base import ExecutorBase
4
+from vllm.logger import init_logger
5
+from vllm.utils import get_distributed_init_method, get_ip, get_open_port
6
+from vllm.worker.worker_base import WorkerWrapperBase
7
+
8
+logger = init_logger(__name__)
9
+
10
+
11
+class UniProcExecutor(ExecutorBase):
12
+
13
+ uses_ray: bool = False
14
+
15
+ def _init_executor(self) -> None:
16
+ """Initialize the worker and load the model.
17
+ """
18
+ self.driver_worker = WorkerWrapperBase(vllm_config=self.vllm_config,
19
+ rank=0)
20
+ distributed_init_method = get_distributed_init_method(
21
+ get_ip(), get_open_port())
22
+ local_rank = 0
23
+ rank = 0
24
+ kwargs = dict(
25
+ vllm_config=self.vllm_config,
26
+ local_rank=local_rank,
27
+ rank=rank,
28
+ distributed_init_method=distributed_init_method,
29
+ is_driver_worker=(not self.parallel_config)
30
+ or (rank % self.parallel_config.tensor_parallel_size == 0),
31
+ )
32
+ self.collective_rpc("init_worker", args=([kwargs], ))
33
+ self.collective_rpc("init_device")
34
+ self.collective_rpc("load_model")
35
+
36
+ def collective_rpc(self,
37
+ method: str,
38
+ timeout: Optional[float] = None,
39
+ args: Tuple = (),
40
+ kwargs: Optional[Dict] = None) -> List[Any]:
41
+ if kwargs is None:
42
+ kwargs = {}
43
+ try:
44
+ func = getattr(self.driver_worker, method)
45
+ except AttributeError:
46
+ raise NotImplementedError(f"Method {method} is not implemented.") \
47
+ from None
48
+ answer = func(*args, **kwargs)
49
+ return [answer]
50
+
51
+ def check_health(self) -> None:
52
+ # UniProcExecutor will always be healthy as long as
53
+ # it's running.
54
+ return
55
+
56
+
57
+UniProcExecutorAsync = UniProcExecutor
vllm/executor/xpu_executor.py
deleted
-39
@@ -1,39 +0,0 @@
1
-from typing import List, Optional, Union
2
-
3
-from vllm.executor.executor_base import ExecutorAsyncBase
4
-from vllm.executor.gpu_executor import GPUExecutor
5
-from vllm.logger import init_logger
6
-from vllm.model_executor.layers.sampler import SamplerOutput
7
-from vllm.sequence import ExecuteModelRequest, PoolerOutput
8
-from vllm.utils import make_async
9
-
10
-logger = init_logger(__name__)
11
-
12
-
13
-class XPUExecutor(GPUExecutor):
14
-
15
- uses_ray: bool = False
16
-
17
- def _init_executor(self) -> None:
18
- assert self.device_config.device_type == "xpu"
19
- assert self.speculative_config is None, (
20
- "Speculative decoding not yet supported for XPU backend")
21
-
22
- GPUExecutor._init_executor(self)
23
-
24
- def execute_model(
25
- self, execute_model_req: ExecuteModelRequest
26
- ) -> Optional[List[Union[SamplerOutput, PoolerOutput]]]:
27
- output = self.driver_worker.execute_model(execute_model_req)
28
- return output
29
-
30
-
31
-class XPUExecutorAsync(XPUExecutor, ExecutorAsyncBase):
32
-
33
- async def execute_model_async(
34
- self,
35
- execute_model_req: ExecuteModelRequest,
36
- ) -> List[SamplerOutput]:
37
- output = await make_async(self.driver_worker.execute_model
38
- )(execute_model_req=execute_model_req)
39
- return output
vllm/platforms/cpu.py
+27
@@ -1,3 +1,4 @@
1
+import os
2
from typing import TYPE_CHECKING, Optional
3
4
import psutil
@@ -105,6 +106,32 @@ class CpuPlatform(Platform):
106
else:
107
parallel_config.worker_cls = "vllm.worker.cpu_worker.CPUWorker"
108
109
+ assert vllm_config.device_config.device_type == "cpu"
110
+
111
+ #
112
+ # Environment variables for CPU executor
113
+ #
114
+
115
+ # Disable torch async compiling which won't work with daemonic processes
116
+ os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
117
+
118
+ # Intel OpenMP setting
119
+ ld_prealod_str = os.getenv("LD_PRELOAD", "")
120
+ if "libiomp5.so" in ld_prealod_str:
121
+ # The time(milliseconds) that a thread should wait after
122
+ # completing the execution of a parallel region, before sleeping.
123
+ os.environ['KMP_BLOCKTIME'] = "1"
124
+ # Prevents the CPU to run into low performance state
125
+ os.environ['KMP_TPAUSE'] = "0"
126
+ # Provides fine granularity parallelism
127
+ os.environ['KMP_FORKJOIN_BARRIER_PATTERN'] = "dist,dist"
128
+ os.environ['KMP_PLAIN_BARRIER_PATTERN'] = "dist,dist"
129
+ os.environ['KMP_REDUCTION_BARRIER_PATTERN'] = "dist,dist"
130
+
131
+ # To hint IPEX uses shared memory based AllReduce
132
+ os.environ["LOCAL_WORLD_SIZE"] = str(
133
+ vllm_config.parallel_config.tensor_parallel_size)
134
+
135
@classmethod
136
def is_pin_memory_available(cls) -> bool:
137
logger.warning("Pin memory is not supported on CPU.")
vllm/platforms/cuda.py
+22
@@ -139,6 +139,28 @@ class CudaPlatformBase(Platform):
139
else:
140
parallel_config.worker_cls = "vllm.worker.worker.Worker"
141
142
+ world_size = parallel_config.world_size
143
+ tensor_parallel_size = parallel_config.tensor_parallel_size
144
+
145
+ from vllm.utils import (cuda_device_count_stateless,
146
+ update_environment_variables)
147
+
148
+ # Set CUDA_VISIBLE_DEVICES for the driver, inherited by workers
149
+ if "CUDA_VISIBLE_DEVICES" not in os.environ:
150
+ update_environment_variables({
151
+ "CUDA_VISIBLE_DEVICES": (",".join(map(str, range(world_size))))
152
+ })
153
+
154
+ cuda_device_count = cuda_device_count_stateless()
155
+ # Use confusing message for more common TP-only case.
156
+ assert tensor_parallel_size <= cuda_device_count, (
157
+ f"please set tensor_parallel_size ({tensor_parallel_size}) "
158
+ f"to less than max local gpu count ({cuda_device_count})")
159
+
160
+ assert world_size <= cuda_device_count, (
161
+ f"please ensure that world_size ({world_size}) "
162
+ f"is less than than max local gpu count ({cuda_device_count})")
163
+
164
cache_config = vllm_config.cache_config
165
if cache_config and cache_config.block_size is None:
166
cache_config.block_size = 16
vllm/platforms/neuron.py
+8
@@ -35,6 +35,14 @@ class NeuronPlatform(Platform):
35
parallel_config.worker_cls = \
36
"vllm.worker.neuron_worker.NeuronWorker"
37
38
+ if parallel_config.world_size > 1:
39
+ parallel_config.distributed_executor_backend = "uni"
40
+
41
+ assert (vllm_config.lora_config is
42
+ None), "LoRA is not supported for Neuron backend."
43
+ assert (not vllm_config.speculative_config
44
+ ), "Speculative decoding not yet supported for Neuron backend."
45
+
46
cache_config = vllm_config.cache_config
47
if cache_config:
48
# neuron needs block_size = max_model_len
vllm/platforms/openvino.py
+9
-3
@@ -66,9 +66,8 @@ class OpenVinoPlatform(Platform):
66
from vllm.utils import GiB_bytes
67
68
parallel_config = vllm_config.parallel_config
69
- assert (
70
- parallel_config.world_size == 1
71
- ), "OpenVINOExecutor only supports single CPU socket currently."
69
+ assert (parallel_config.world_size == 1
70
+ ), "OpenVINO only supports single CPU socket currently."
71
72
if parallel_config.worker_cls == "auto":
73
parallel_config.worker_cls = \
@@ -141,3 +140,10 @@ class OpenVinoPlatform(Platform):
140
raise RuntimeError(
141
"Invalid environment variable VLLM_OPENVINO_KVCACHE_SPACE"
142
f" {kv_cache_space}, expect a positive integer value.")
143
+
144
+ assert vllm_config.device_config.device_type == "openvino"
145
+ assert vllm_config.lora_config is None, \
146
+ "OpenVINO backend doesn't support LoRA"
147
+ assert cls.is_openvino_cpu() or \
148
+ cls.is_openvino_gpu(), \
149
+ "OpenVINO backend supports only CPU and GPU devices"
vllm/platforms/tpu.py
+10
@@ -72,6 +72,16 @@ class TpuPlatform(Platform):
72
assert vllm_config.speculative_config is None, \
73
"TPU does not support speculative decoding"
74
75
+ assert not vllm_config.scheduler_config.chunked_prefill_enabled, (
76
+ "Chunked prefill is not yet supported for TPU backend")
77
+ assert not vllm_config.speculative_config, (
78
+ "Speculative decoding is not yet supported for TPU backend")
79
+ if vllm_config.model_config.dtype in (torch.float16, torch.float32):
80
+ logger.warning(
81
+ "The TPU backend currently does not support %s. "
82
+ "Using bfloat16 instead.", vllm_config.model_config.dtype)
83
+ vllm_config.model_config.dtype = torch.bfloat16
84
+
85
parallel_config = vllm_config.parallel_config
86
scheduler_config = vllm_config.scheduler_config
87
if parallel_config.worker_cls == "auto":
vllm/platforms/xpu.py
+18
-4
@@ -78,17 +78,31 @@ class XPUPlatform(Platform):
78
raise NotImplementedError(
79
"XPU does not support speculative decoding")
80
81
+ if vllm_config.device_config is not None:
82
+ assert vllm_config.device_config.device_type == "xpu"
83
+
84
# check and update parallel config
85
parallel_config = vllm_config.parallel_config
83
- if (parallel_config.distributed_executor_backend is not None
84
- and parallel_config.distributed_executor_backend != "ray"):
86
+ if parallel_config.worker_cls == "auto":
87
+ parallel_config.worker_cls = "vllm.worker.xpu_worker.XPUWorker"
88
+
89
+ if parallel_config.distributed_executor_backend is None:
90
+ parallel_config.distributed_executor_backend = "ray"
91
+ elif parallel_config.distributed_executor_backend == "mp":
92
+ # FIXME(kunshang):
93
+ # spawn needs calling `if __name__ == '__main__':``
94
+ # fork is not supported for xpu start new process.
95
+ logger.error(
96
+ "Both start methods (spawn and fork) have issue "
97
+ "on XPU if you use mp backend, setting it to ray instead.")
98
+ parallel_config.distributed_executor_backend = "ray"
99
+
100
+ elif parallel_config.distributed_executor_backend != "ray":
101
logger.warning(
102
"%s is not supported on XPU, fallback to ray distributed"
103
" executor backend.",
104
parallel_config.distributed_executor_backend)
105
parallel_config.distributed_executor_backend = "ray"
90
- if parallel_config.worker_cls == "auto":
91
- parallel_config.worker_cls = "vllm.worker.xpu_worker.XPUWorker"
106
107
@classmethod
108
def is_pin_memory_available(cls):
vllm/spec_decode/medusa_worker.py
+3
-5
@@ -9,17 +9,15 @@ from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata
9
from vllm.spec_decode.interfaces import SpeculativeProposals
10
from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
11
from vllm.spec_decode.top1_proposer import Top1Proposer
12
-from vllm.worker.worker_base import WorkerWrapperBase
12
+from vllm.worker.worker_base import DelegateWorkerBase
13
14
15
-class MedusaWorker(NonLLMProposerWorkerBase, WorkerWrapperBase):
15
+class MedusaWorker(NonLLMProposerWorkerBase, DelegateWorkerBase):
16
"""Worker for Medusa.
17
"""
18
19
def __init__(self, *args, **kwargs):
20
- super().__init__(kwargs.get("vllm_config"))
21
- self.init_worker(*args, **kwargs)
22
-
20
+ DelegateWorkerBase.__init__(self, *args, **kwargs)
21
# Lazy initialization list.
22
self._proposer: Top1Proposer
23
vllm/spec_decode/multi_step_worker.py
+3
-18
@@ -16,10 +16,10 @@ from vllm.spec_decode.interfaces import (SpeculativeProposals,
16
SpeculativeProposer)
17
from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
18
from vllm.spec_decode.top1_proposer import Top1Proposer
19
-from vllm.worker.worker_base import WorkerWrapperBase
19
+from vllm.worker.worker_base import DelegateWorkerBase
20
21
22
-class MultiStepWorker(ProposerWorkerBase, WorkerWrapperBase):
22
+class MultiStepWorker(ProposerWorkerBase, DelegateWorkerBase):
23
"""The MultiStepWorker is equivalent to a Worker except that it allows
24
multiple forward passes in a single call, assuming the scheduler has
25
allocated enough space to store the additional KV. This reduces overhead
@@ -32,15 +32,12 @@ class MultiStepWorker(ProposerWorkerBase, WorkerWrapperBase):
32
"""
33
34
def __init__(self, *args, **kwargs):
35
- super().__init__(kwargs.get("vllm_config"))
36
- self.init_worker(*args, **kwargs)
37
-
35
+ DelegateWorkerBase.__init__(self, *args, **kwargs)
36
# Lazy initialization list.
37
self._proposer: SpeculativeProposer
38
39
def init_device(self) -> None:
40
self.worker.init_device()
43
-
41
self._proposer = Top1Proposer(
42
weakref.proxy(self), # type: ignore[arg-type]
43
self.device,
@@ -56,18 +53,6 @@ class MultiStepWorker(ProposerWorkerBase, WorkerWrapperBase):
53
self.model_runner.model.sampler.should_modify_greedy_probs_inplace = (
54
True)
55
59
- def determine_num_available_blocks(self) -> Tuple[int, int]:
60
- return self.worker.determine_num_available_blocks()
61
-
62
- def get_cache_block_size_bytes(self) -> int:
63
- return self.worker.get_cache_block_size_bytes()
64
-
65
- def initialize_cache(self, *args, **kwargs) -> None:
66
- self.worker.initialize_cache(*args, **kwargs)
67
-
68
- def execute_model(self, *args, **kwargs) -> List[SamplerOutput]:
69
- return self.worker.execute_model(*args, **kwargs)
70
-
56
@torch.inference_mode()
57
def sampler_output(
58
self,
vllm/spec_decode/spec_decode_worker.py
+5
-4
@@ -40,8 +40,8 @@ from vllm.spec_decode.util import (Timer, create_logprobs_output,
40
get_all_num_logprobs,
41
get_sampled_token_logprobs, nvtx_range,
42
split_batch_by_proposal_len)
43
-from vllm.worker.worker_base import (LoraNotSupportedWorkerBase, WorkerBase,
44
- WorkerWrapperBase)
43
+from vllm.utils import resolve_obj_by_qualname
44
+from vllm.worker.worker_base import LoraNotSupportedWorkerBase, WorkerBase
45
46
logger = init_logger(__name__)
47
@@ -64,8 +64,9 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
64
target_worker_config = copy.deepcopy(vllm_config)
65
target_worker_config.parallel_config.worker_cls =\
66
target_worker_config.parallel_config.sd_worker_cls
67
- target_worker = WorkerWrapperBase(vllm_config=target_worker_config)
68
- target_worker.init_worker(*args, **kwargs)
67
+ cls = resolve_obj_by_qualname(
68
+ target_worker_config.parallel_config.worker_cls)
69
+ target_worker = cls(*args, **kwargs)
70
# Set the disable_logprobs variable in the TargetModelRunner instance
71
# as per its value specified in the SpeculativeConfig.
72
target_worker.model_runner.disable_logprobs =\
vllm/v1/executor/abstract.py
+3
-2
@@ -14,8 +14,9 @@ class Executor(ABC):
14
distributed_executor_backend = (
15
vllm_config.parallel_config.distributed_executor_backend)
16
if distributed_executor_backend == "ray":
17
- from vllm.v1.executor.ray_executor import RayExecutor
18
- executor_class = RayExecutor
17
+ from vllm.executor.ray_distributed_executor import ( # noqa
18
+ RayDistributedExecutor)
19
+ executor_class = RayDistributedExecutor
20
elif distributed_executor_backend == "mp":
21
from vllm.v1.executor.multiproc_executor import MultiprocExecutor
22
executor_class = MultiprocExecutor
vllm/v1/executor/multiproc_executor.py
+13
-4
@@ -246,9 +246,18 @@ class WorkerProc:
246
ready_path: str,
247
):
248
self.rank = rank
249
- wrapper = WorkerWrapperBase(vllm_config=vllm_config)
250
- wrapper.init_worker(vllm_config, local_rank, rank,
251
- distributed_init_method)
249
+ wrapper = WorkerWrapperBase(vllm_config=vllm_config, rank=rank)
250
+ # TODO: move `init_worker` to executor level as a collective rpc call
251
+ all_kwargs: List[Dict] = [
252
+ {} for _ in range(vllm_config.parallel_config.world_size)
253
+ ]
254
+ all_kwargs[rank] = {
255
+ "vllm_config": vllm_config,
256
+ "local_rank": local_rank,
257
+ "rank": rank,
258
+ "distributed_init_method": distributed_init_method,
259
+ }
260
+ wrapper.init_worker(all_kwargs)
261
self.worker = wrapper.worker
262
263
pid = os.getpid()
@@ -270,7 +279,7 @@ class WorkerProc:
279
ready_socket.send_string(WorkerProc.READY_STR)
280
ready_socket.send(payload)
281
273
- self.worker.initialize()
282
+ self.worker.init_device()
283
self.worker.load_model()
284
285
@staticmethod
vllm/v1/executor/uniproc_executor.py
+1
-1
@@ -27,7 +27,7 @@ class UniprocExecutor(Executor):
27
self.observability_config = vllm_config.observability_config
28
29
self.worker: Worker = self._create_worker()
30
- self.worker.initialize()
30
+ self.worker.init_device()
31
self.worker.load_model()
32
33
def _create_worker(
vllm/v1/worker/gpu_worker.py
+2
-1
@@ -33,6 +33,7 @@ class Worker:
33
local_rank: int,
34
rank: int,
35
distributed_init_method: str,
36
+ is_driver_worker: bool = False,
37
):
38
39
# TODO: use WorkerBase.__init__(self, vllm_config=vllm_config)
@@ -75,7 +76,7 @@ class Worker:
76
else:
77
self.profiler = None
78
78
- def initialize(self):
79
+ def init_device(self):
80
if self.device_config.device.type == "cuda":
81
# torch.distributed.all_reduce does not free the input tensor until
82
# the synchronization point. This causes the memory usage to grow
vllm/worker/hpu_worker.py
+66
@@ -2,6 +2,7 @@
2
# Copyright (C) 2024 Habana Labs, Ltd. an Intel Company
3
###############################################################################
4
5
+import contextlib
6
import gc
7
import os
8
from typing import List, Optional, Set, Tuple, Type
@@ -18,6 +19,7 @@ from vllm.distributed import (ensure_model_parallel_initialized,
19
from vllm.logger import init_logger
20
from vllm.lora.request import LoRARequest
21
from vllm.model_executor import set_random_seed
22
+from vllm.model_executor.layers.sampler import SamplerOutput
23
from vllm.prompt_adapter.request import PromptAdapterRequest
24
from vllm.sequence import ExecuteModelRequest
25
from vllm.utils import bind_kv_cache
@@ -124,6 +126,70 @@ class HPUWorker(LocalOrDistributedWorkerBase):
126
def load_model(self):
127
self.model_runner.load_model()
128
129
+ def execute_model(
130
+ self,
131
+ execute_model_req: Optional[ExecuteModelRequest] = None,
132
+ ) -> Optional[List[SamplerOutput]]:
133
+ assert execute_model_req is not None
134
+ # VLLM_HPU_LOG_STEP_GRAPH_COMPILATION - will log graph compilations per engine step, only when there was any - highly recommended to use alongside PT_HPU_METRICS_GC_DETAILS! # noqa:E501
135
+ # VLLM_HPU_LOG_STEP_GRAPH_COMPILATION_ALL - will log graph compilations per engine step, always, even if there were none # noqa:E501
136
+ # VLLM_HPU_LOG_STEP_CPU_FALLBACKS - will log cpu fallbacks per engine step, only when there was any # noqa:E501
137
+ # VLLM_HPU_LOG_STEP_CPU_FALLBACKS_ALL - will log cpu fallbacks per engine step, always, even if there were none # noqa:E501
138
+ log_graph_compilation_all = os.environ.get(
139
+ 'VLLM_HPU_LOG_STEP_GRAPH_COMPILATION_ALL', '0') != '0'
140
+ log_graph_compilation = os.environ.get(
141
+ 'VLLM_HPU_LOG_STEP_GRAPH_COMPILATION',
142
+ '0') != '0' or log_graph_compilation_all
143
+ log_cpu_fallbacks_all = os.environ.get(
144
+ 'VLLM_HPU_LOG_STEP_CPU_FALLBACKS_ALL', '0') != '0'
145
+ log_cpu_fallbacks = os.environ.get('VLLM_HPU_LOG_STEP_CPU_FALLBACKS',
146
+ '0') != '0' or log_cpu_fallbacks_all
147
+ if log_graph_compilation or log_cpu_fallbacks:
148
+ from habana_frameworks.torch.hpu.metrics import metric_localcontext
149
+ seq_group_metadata_list = execute_model_req.seq_group_metadata_list
150
+ is_prompt = any([
151
+ seq_group_metadata.is_prompt
152
+ for seq_group_metadata in seq_group_metadata_list
153
+ ])
154
+ max_context_len = max([
155
+ max([
156
+ len(v.prompt_token_ids) + len(v.output_token_ids)
157
+ for v in seq_group_metadata.seq_data.values()
158
+ ]) for seq_group_metadata in seq_group_metadata_list
159
+ ]) # whoa, that's some spicy stuff right here
160
+ max_num_blocks = (
161
+ (max_context_len - 1) // self.cache_config.block_size) + 1
162
+ input_stats = (f'is_prompt: {is_prompt}, '
163
+ f'num_seqs: {len(seq_group_metadata_list)}, '
164
+ f'max_context_len: {max_context_len}, '
165
+ f'max_num_blocks {max_num_blocks}')
166
+ gc_ctx = metric_localcontext(
167
+ "graph_compilation"
168
+ ) if log_graph_compilation else contextlib.nullcontext()
169
+ cpu_fallback_ctx = metric_localcontext(
170
+ "cpu_fallback"
171
+ ) if log_cpu_fallbacks else contextlib.nullcontext()
172
+ with gc_ctx as gc_local_metric, \
173
+ cpu_fallback_ctx as cpu_fallback_local_metric:
174
+ output = LocalOrDistributedWorkerBase.execute_model(
175
+ self, execute_model_req)
176
+ if (log_graph_compilation and gc_local_metric.stats()[0][1] > 0
177
+ ) or log_graph_compilation_all:
178
+ msg = ("VLLM_HPU_STEP_GRAPH_COMPILATION: "
179
+ f"{gc_local_metric.stats()}, {input_stats}")
180
+ logger.warning(msg)
181
+ if (log_cpu_fallbacks and cpu_fallback_local_metric.stats()[0][1] >
182
+ 0) or log_cpu_fallbacks_all:
183
+ msg = ("VLLM_HPU_STEP_CPU_FALLBACK: "
184
+ f"{cpu_fallback_local_metric.stats()}, {input_stats}")
185
+ logger.warning(msg)
186
+
187
+ return output
188
+
189
+ output = LocalOrDistributedWorkerBase.execute_model(
190
+ self, execute_model_req)
191
+ return output
192
+
193
@torch.inference_mode()
194
def determine_num_available_blocks(self) -> Tuple[int, int]:
195
"""Profiles the peak memory usage of the model to determine how many
vllm/worker/neuron_worker.py
+23
-5
@@ -8,6 +8,7 @@ from vllm.config import VllmConfig
8
from vllm.distributed import (ensure_model_parallel_initialized,
9
init_distributed_environment)
10
from vllm.model_executor import set_random_seed
11
+from vllm.model_executor.layers.sampler import SamplerOutput
12
from vllm.sequence import ExecuteModelRequest
13
from vllm.worker.neuron_model_runner import NeuronModelRunner
14
from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
@@ -25,6 +26,7 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
26
local_rank: int,
27
rank: int,
28
distributed_init_method: str,
29
+ is_driver_worker: bool = True,
30
) -> None:
31
WorkerBase.__init__(self, vllm_config=vllm_config)
32
self.local_rank = local_rank
@@ -37,7 +39,22 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
39
40
self.model_runner: NeuronModelRunner = NeuronModelRunner(
41
vllm_config=vllm_config)
40
- self.is_driver_worker = True
42
+ self.is_driver_worker = is_driver_worker
43
+
44
+ def execute_model(
45
+ self,
46
+ execute_model_req: Optional[ExecuteModelRequest] = None,
47
+ ) -> Optional[List[SamplerOutput]]:
48
+ assert execute_model_req is not None
49
+ assert (not execute_model_req.blocks_to_swap_in
50
+ and not execute_model_req.blocks_to_swap_out
51
+ and not execute_model_req.blocks_to_copy), (
52
+ "Cache operations are not supported for Neuron backend.")
53
+ assert execute_model_req.num_lookahead_slots == 0, (
54
+ "lookahead not supported for Neuron backend.")
55
+ output = LocalOrDistributedWorkerBase.execute_model(
56
+ self, execute_model_req)
57
+ return output
58
59
def init_device(self) -> None:
60
self.init_distributed_environment()
@@ -103,13 +120,14 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
120
121
def init_distributed_environment(self):
122
"""Neuron uses transformers-neuronx for tensor parallelism.
106
-
107
- vLLM still needs the environment inited when TP/PP > 1
123
+ It has only one process to control multiple devices.
124
+ vLLM still needs the environment initialized when TP/PP > 1,
125
+ so we initialize a distributed environment with one process.
126
"""
127
init_distributed_environment(
128
world_size=1,
111
- rank=self.rank,
112
- local_rank=self.local_rank,
129
+ rank=0,
130
+ local_rank=0,
131
distributed_init_method=self.distributed_init_method,
132
backend="gloo",
133
)
vllm/worker/openvino_worker.py
+2
-4
@@ -211,16 +211,14 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
211
212
def __init__(
213
self,
214
- ov_core: ov.Core,
214
vllm_config: VllmConfig,
215
local_rank: int,
216
rank: int,
217
distributed_init_method: str,
219
- kv_cache_dtype: Optional[ov.Type] = ov.Type.undefined,
218
is_driver_worker: bool = False,
219
) -> None:
222
- self.ov_core = ov_core
220
WorkerBase.__init__(self, vllm_config)
221
+ self.ov_core = ov.Core()
222
self.parallel_config.rank = rank
223
self.local_rank = local_rank
224
self.rank = rank
@@ -237,7 +235,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
235
self.model_runner = OpenVINOModelRunner(
236
self.ov_core,
237
vllm_config=self.vllm_config,
240
- kv_cache_dtype=kv_cache_dtype,
238
+ kv_cache_dtype=self.vllm_config.cache_config.cache_dtype,
239
is_driver_worker=is_driver_worker,
240
)
241
# Uninitialized cache engine. Will be initialized by
vllm/worker/worker_base.py
+76
-10
@@ -88,7 +88,6 @@ class WorkerBase(ABC):
88
if output is None:
89
return None
90
91
- @abstractmethod
91
def execute_model(
92
self,
93
execute_model_req: Optional[ExecuteModelRequest] = None
@@ -119,6 +118,58 @@ class WorkerBase(ABC):
118
raise NotImplementedError
119
120
121
+class DelegateWorkerBase(WorkerBase):
122
+ """
123
+ A class that delegates all methods to another WorkerBase instance. This is
124
+ useful for creating a WorkerBase that wraps another WorkerBase instance,
125
+ e.g. speculative decoding.
126
+ """
127
+ worker: WorkerBase
128
+
129
+ def __init__(
130
+ self,
131
+ *args,
132
+ **kwargs,
133
+ ) -> None:
134
+ vllm_config: VllmConfig = kwargs.get("vllm_config")
135
+ cls = resolve_obj_by_qualname(vllm_config.parallel_config.worker_cls)
136
+ self.worker = cls(*args, **kwargs)
137
+
138
+ def init_device(self) -> None:
139
+ self.worker.init_device()
140
+
141
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
142
+ return self.worker.determine_num_available_blocks()
143
+
144
+ def initialize_cache(self, num_gpu_blocks: int,
145
+ num_cpu_blocks: int) -> None:
146
+ self.worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
147
+
148
+ def execute_model(
149
+ self,
150
+ execute_model_req: Optional[ExecuteModelRequest] = None
151
+ ) -> Optional[List[SamplerOutput]]:
152
+ return self.worker.execute_model(execute_model_req)
153
+
154
+ def get_cache_block_size_bytes(self) -> int:
155
+ return self.worker.get_cache_block_size_bytes()
156
+
157
+ def add_lora(self, lora_request: LoRARequest) -> bool:
158
+ return self.worker.add_lora(lora_request)
159
+
160
+ def remove_lora(self, lora_id: int) -> bool:
161
+ return self.worker.remove_lora(lora_id)
162
+
163
+ def pin_lora(self, lora_id: int) -> bool:
164
+ return self.worker.pin_lora(lora_id)
165
+
166
+ def list_loras(self) -> Set[int]:
167
+ return self.worker.list_loras()
168
+
169
+ def __getattr__(self, attr):
170
+ return getattr(self.worker, attr)
171
+
172
+
173
class LoraNotSupportedWorkerBase(WorkerBase):
174
"""Partial implementation of WorkerBase that raises exceptions when LoRA
175
methods are invoked.
@@ -419,17 +470,31 @@ class WorkerWrapperBase:
470
def __init__(
471
self,
472
vllm_config: VllmConfig,
473
+ rank: int = 0,
474
) -> None:
475
+ self.rank = rank
476
self.vllm_config = vllm_config
424
- trust_remote_code = vllm_config.model_config.trust_remote_code
477
self.worker: Optional[WorkerBase] = None
426
- if trust_remote_code:
427
- # note: lazy import to avoid importing torch before initializing
428
- from vllm.utils import init_cached_hf_modules
429
- init_cached_hf_modules()
478
+ if vllm_config.model_config is not None:
479
+ # it can be None in tests
480
+ trust_remote_code = vllm_config.model_config.trust_remote_code
481
+ if trust_remote_code:
482
+ # note: lazy import to avoid importing torch before initializing
483
+ from vllm.utils import init_cached_hf_modules
484
+ init_cached_hf_modules()
485
+
486
+ def adjust_rank(self, rank_mapping: Dict[int, int]) -> None:
487
+ """
488
+ Adjust the rank based on the given mapping.
489
+ It is only used during the initialization of the executor,
490
+ to adjust the rank of workers after we create all workers.
491
+ """
492
+ if self.rank in rank_mapping:
493
+ self.rank = rank_mapping[self.rank]
494
431
- @staticmethod
432
- def update_environment_variables(envs: Dict[str, str]) -> None:
495
+ def update_environment_variables(self, envs_list: List[Dict[str,
496
+ str]]) -> None:
497
+ envs = envs_list[self.rank]
498
key = 'CUDA_VISIBLE_DEVICES'
499
if key in envs and key in os.environ:
500
# overwriting CUDA_VISIBLE_DEVICES is desired behavior
@@ -437,11 +502,12 @@ class WorkerWrapperBase:
502
del os.environ[key]
503
update_environment_variables(envs)
504
440
- def init_worker(self, *args, **kwargs):
505
+ def init_worker(self, all_kwargs: List[Dict[str, Any]]) -> None:
506
"""
507
Here we inject some common logic before initializing the worker.
508
Arguments are passed to the worker class constructor.
509
"""
510
+ kwargs = all_kwargs[self.rank]
511
enable_trace_function_call_for_thread(self.vllm_config)
512
513
# see https://github.com/NVIDIA/nccl/issues/1234
@@ -452,7 +518,7 @@ class WorkerWrapperBase:
518
519
worker_class = resolve_obj_by_qualname(
520
self.vllm_config.parallel_config.worker_cls)
455
- self.worker = worker_class(*args, **kwargs)
521
+ self.worker = worker_class(**kwargs)
522
assert self.worker is not None
523
524
def execute_method(self, method: str, *args, **kwargs):