A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Core] Allow specifying custom Executor (#6557)
Antoni Baum committed
Jul 19, 2024 at 18:25 UTC
7bd82002ae0e7ee8c4e5da0d43cfe0fd85372b4a
22 files changed
+310
-92
tests/conftest.py
+4
@@ -564,6 +564,10 @@ def get_tokenizer_pool_config(tokenizer_group_type):
564
return TokenizerPoolConfig(pool_size=1,
565
pool_type="ray",
566
extra_config={})
567
+ if isinstance(tokenizer_group_type, type):
568
+ return TokenizerPoolConfig(pool_size=1,
569
+ pool_type=tokenizer_group_type,
570
+ extra_config={})
571
raise ValueError(f"Unknown tokenizer_group_type: {tokenizer_group_type}")
572
573
tests/engine/test_custom_executor.py
new
+91
@@ -0,0 +1,91 @@
1
+import asyncio
2
+import os
3
+
4
+import pytest
5
+
6
+from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
7
+from vllm.engine.async_llm_engine import AsyncLLMEngine
8
+from vllm.engine.llm_engine import LLMEngine
9
+from vllm.executor.gpu_executor import GPUExecutor, GPUExecutorAsync
10
+from vllm.sampling_params import SamplingParams
11
+
12
+
13
+class Mock:
14
+ ...
15
+
16
+
17
+class CustomGPUExecutor(GPUExecutor):
18
+
19
+ def execute_model(self, *args, **kwargs):
20
+ # Drop marker to show that this was ran
21
+ with open(".marker", "w"):
22
+ ...
23
+ return super().execute_model(*args, **kwargs)
24
+
25
+
26
+class CustomGPUExecutorAsync(GPUExecutorAsync):
27
+
28
+ async def execute_model_async(self, *args, **kwargs):
29
+ with open(".marker", "w"):
30
+ ...
31
+ return await super().execute_model_async(*args, **kwargs)
32
+
33
+
34
+@pytest.mark.parametrize("model", ["facebook/opt-125m"])
35
+def test_custom_executor_type_checking(model):
36
+ with pytest.raises(ValueError):
37
+ engine_args = EngineArgs(model=model,
38
+ distributed_executor_backend=Mock)
39
+ LLMEngine.from_engine_args(engine_args)
40
+ with pytest.raises(ValueError):
41
+ engine_args = AsyncEngineArgs(model=model,
42
+ distributed_executor_backend=Mock)
43
+ AsyncLLMEngine.from_engine_args(engine_args)
44
+ with pytest.raises(TypeError):
45
+ engine_args = AsyncEngineArgs(
46
+ model=model, distributed_executor_backend=CustomGPUExecutor)
47
+ AsyncLLMEngine.from_engine_args(engine_args)
48
+
49
+
50
+@pytest.mark.parametrize("model", ["facebook/opt-125m"])
51
+def test_custom_executor(model, tmpdir):
52
+ cwd = os.path.abspath(".")
53
+ os.chdir(tmpdir)
54
+ try:
55
+ assert not os.path.exists(".marker")
56
+
57
+ engine_args = EngineArgs(
58
+ model=model, distributed_executor_backend=CustomGPUExecutor)
59
+ engine = LLMEngine.from_engine_args(engine_args)
60
+ sampling_params = SamplingParams(max_tokens=1)
61
+
62
+ engine.add_request("0", "foo", sampling_params)
63
+ engine.step()
64
+
65
+ assert os.path.exists(".marker")
66
+ finally:
67
+ os.chdir(cwd)
68
+
69
+
70
+@pytest.mark.parametrize("model", ["facebook/opt-125m"])
71
+def test_custom_executor_async(model, tmpdir):
72
+ cwd = os.path.abspath(".")
73
+ os.chdir(tmpdir)
74
+ try:
75
+ assert not os.path.exists(".marker")
76
+
77
+ engine_args = AsyncEngineArgs(
78
+ model=model, distributed_executor_backend=CustomGPUExecutorAsync)
79
+ engine = AsyncLLMEngine.from_engine_args(engine_args)
80
+ sampling_params = SamplingParams(max_tokens=1)
81
+
82
+ async def t():
83
+ stream = await engine.add_request("0", "foo", sampling_params)
84
+ async for x in stream:
85
+ ...
86
+
87
+ asyncio.run(t())
88
+
89
+ assert os.path.exists(".marker")
90
+ finally:
91
+ os.chdir(cwd)
tests/tokenization/test_tokenizer_group.py
+17
-4
@@ -7,17 +7,28 @@ from unittest.mock import patch
7
import pytest
8
from transformers import AutoTokenizer, PreTrainedTokenizerBase
9
10
-from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
10
+from vllm.transformers_utils.tokenizer_group import (TokenizerGroup,
11
+ get_tokenizer_group)
12
from vllm.transformers_utils.tokenizer_group.ray_tokenizer_group import (
13
RayTokenizerGroupPool)
13
-from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
14
- TokenizerGroup)
14
15
from ..conftest import get_tokenizer_pool_config
16
17
18
+class CustomTokenizerGroup(TokenizerGroup):
19
+
20
+ def __init__(self, *args, **kwargs):
21
+ super().__init__(*args, **kwargs)
22
+ self._i = 0
23
+
24
+ def encode(self, *args, **kwargs):
25
+ self._i += 1
26
+ return super().encode(*args, **kwargs)
27
+
28
+
29
@pytest.mark.asyncio
20
-@pytest.mark.parametrize("tokenizer_group_type", [None, "ray"])
30
+@pytest.mark.parametrize("tokenizer_group_type",
31
+ [None, "ray", CustomTokenizerGroup])
32
async def test_tokenizer_group(tokenizer_group_type):
33
reference_tokenizer = AutoTokenizer.from_pretrained("gpt2")
34
tokenizer_group = get_tokenizer_group(
@@ -36,6 +47,8 @@ async def test_tokenizer_group(tokenizer_group_type):
47
PreTrainedTokenizerBase)
48
assert tokenizer_group.get_lora_tokenizer(
49
None) == await tokenizer_group.get_lora_tokenizer_async(None)
50
+ if tokenizer_group_type is CustomTokenizerGroup:
51
+ assert tokenizer_group._i > 0
52
53
54
@pytest.mark.asyncio
vllm/config.py
+28
-11
@@ -1,7 +1,7 @@
1
import enum
2
import json
3
from dataclasses import dataclass, field, fields
4
-from typing import TYPE_CHECKING, ClassVar, List, Optional, Tuple, Union
4
+from typing import TYPE_CHECKING, ClassVar, List, Optional, Tuple, Type, Union
5
6
import torch
7
from transformers import PretrainedConfig
@@ -18,7 +18,10 @@ from vllm.utils import (cuda_device_count_stateless, get_cpu_memory, is_cpu,
18
if TYPE_CHECKING:
19
from ray.util.placement_group import PlacementGroup
20
21
+ from vllm.executor.executor_base import ExecutorBase
22
from vllm.model_executor.model_loader.loader import BaseModelLoader
23
+ from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
24
+ BaseTokenizerGroup)
25
26
logger = init_logger(__name__)
27
@@ -527,11 +530,12 @@ class TokenizerPoolConfig:
530
pool type.
531
"""
532
pool_size: int
530
- pool_type: str
533
+ pool_type: Union[str, Type["BaseTokenizerGroup"]]
534
extra_config: dict
535
536
def __post_init__(self):
534
- if self.pool_type not in ("ray", ):
537
+ if self.pool_type not in ("ray", ) and not isinstance(
538
+ self.pool_type, type):
539
raise ValueError(f"Unknown pool type: {self.pool_type}")
540
if not isinstance(self.extra_config, dict):
541
raise ValueError("extra_config must be a dictionary.")
@@ -661,7 +665,8 @@ class ParallelConfig:
665
tokenizer_pool_config: Optional[TokenizerPoolConfig] = None,
666
ray_workers_use_nsight: bool = False,
667
placement_group: Optional["PlacementGroup"] = None,
664
- distributed_executor_backend: Optional[str] = None,
668
+ distributed_executor_backend: Optional[Union[
669
+ str, Type["ExecutorBase"]]] = None,
670
) -> None:
671
self.pipeline_parallel_size = pipeline_parallel_size
672
self.tensor_parallel_size = tensor_parallel_size
@@ -676,7 +681,7 @@ class ParallelConfig:
681
if worker_use_ray:
682
if self.distributed_executor_backend is None:
683
self.distributed_executor_backend = "ray"
679
- elif self.distributed_executor_backend != "ray":
684
+ elif not self.use_ray:
685
raise ValueError(f"worker-use-ray can't be used with "
686
f"distributed executor backend "
687
f"'{self.distributed_executor_backend}'.")
@@ -711,12 +716,25 @@ class ParallelConfig:
716
self._verify_args()
717
self.rank = 0
718
719
+ @property
720
+ def use_ray(self) -> bool:
721
+ return self.distributed_executor_backend == "ray" or (
722
+ isinstance(self.distributed_executor_backend, type)
723
+ and self.distributed_executor_backend.uses_ray)
724
+
725
def _verify_args(self) -> None:
715
- if self.distributed_executor_backend not in ("ray", "mp", None):
726
+ # Lazy import to avoid circular import
727
+ from vllm.executor.executor_base import ExecutorBase
728
+
729
+ if self.distributed_executor_backend not in (
730
+ "ray", "mp", None) and not (isinstance(
731
+ self.distributed_executor_backend, type) and issubclass(
732
+ self.distributed_executor_backend, ExecutorBase)):
733
raise ValueError(
717
- "Unrecognized distributed executor backend. Supported values "
718
- "are 'ray' or 'mp'.")
719
- if self.distributed_executor_backend == "ray":
734
+ "Unrecognized distributed executor backend "
735
+ f"{self.distributed_executor_backend}. Supported "
736
+ "values are 'ray', 'mp' or custom ExecutorBase subclass.")
737
+ if self.use_ray:
738
from vllm.executor import ray_utils
739
ray_utils.assert_ray_available()
740
if is_hip():
@@ -724,8 +742,7 @@ class ParallelConfig:
742
logger.info(
743
"Disabled the custom all-reduce kernel because it is not "
744
"supported on AMD GPUs.")
727
- if self.ray_workers_use_nsight and (
728
- not self.distributed_executor_backend == "ray"):
745
+ if self.ray_workers_use_nsight and not self.use_ray:
746
raise ValueError("Unable to use nsight profiling unless workers "
747
"run with Ray.")
748
vllm/engine/arg_utils.py
+15
-3
@@ -2,16 +2,21 @@ import argparse
2
import dataclasses
3
import json
4
from dataclasses import dataclass
5
-from typing import List, Optional, Tuple, Union
5
+from typing import TYPE_CHECKING, List, Optional, Tuple, Type, Union
6
7
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
8
EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
9
MultiModalConfig, ObservabilityConfig, ParallelConfig,
10
PromptAdapterConfig, SchedulerConfig,
11
SpeculativeConfig, TokenizerPoolConfig)
12
+from vllm.executor.executor_base import ExecutorBase
13
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
14
from vllm.utils import FlexibleArgumentParser
15
16
+if TYPE_CHECKING:
17
+ from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
18
+ BaseTokenizerGroup)
19
+
20
21
def nullable_str(val: str):
22
if not val or val == "None":
@@ -36,7 +41,11 @@ class EngineArgs:
41
seed: int = 0
42
max_model_len: Optional[int] = None
43
worker_use_ray: bool = False
39
- distributed_executor_backend: Optional[str] = None
44
+ # Note: Specifying a custom executor backend by passing a class
45
+ # is intended for expert use only. The API may change without
46
+ # notice.
47
+ distributed_executor_backend: Optional[Union[str,
48
+ Type[ExecutorBase]]] = None
49
pipeline_parallel_size: int = 1
50
tensor_parallel_size: int = 1
51
max_parallel_loading_workers: Optional[int] = None
@@ -62,7 +71,10 @@ class EngineArgs:
71
max_seq_len_to_capture: int = 8192
72
disable_custom_all_reduce: bool = False
73
tokenizer_pool_size: int = 0
65
- tokenizer_pool_type: str = "ray"
74
+ # Note: Specifying a tokenizer pool by passing a class
75
+ # is intended for expert use only. The API may change without
76
+ # notice.
77
+ tokenizer_pool_type: Union[str, Type["BaseTokenizerGroup"]] = "ray"
78
tokenizer_pool_extra_config: Optional[dict] = None
79
enable_lora: bool = False
80
max_loras: int = 1
vllm/engine/async_llm_engine.py
+34
-19
@@ -7,12 +7,13 @@ from typing import (AsyncIterator, Callable, Dict, Iterable, List, Optional,
7
from transformers import PreTrainedTokenizer
8
9
import vllm.envs as envs
10
-from vllm.config import DecodingConfig, ModelConfig
10
+from vllm.config import DecodingConfig, EngineConfig, ModelConfig
11
from vllm.core.scheduler import SchedulerOutputs
12
from vllm.engine.arg_utils import AsyncEngineArgs
13
from vllm.engine.async_timeout import asyncio_timeout
14
from vllm.engine.llm_engine import LLMEngine
15
from vllm.engine.metrics import StatLoggerBase
16
+from vllm.executor.executor_base import ExecutorAsyncBase
17
from vllm.executor.ray_utils import initialize_ray_cluster, ray
18
from vllm.inputs import LLMInputs, PromptInputs
19
from vllm.logger import init_logger
@@ -385,25 +386,19 @@ class AsyncLLMEngine:
386
self._request_tracker: RequestTracker
387
388
@classmethod
388
- def from_engine_args(
389
- cls,
390
- engine_args: AsyncEngineArgs,
391
- start_engine_loop: bool = True,
392
- usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
393
- stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
394
- ) -> "AsyncLLMEngine":
395
- """Creates an async LLM engine from the engine arguments."""
396
- # Create the engine configs.
397
- engine_config = engine_args.create_engine_config()
398
-
399
- if engine_args.engine_use_ray:
400
- from vllm.executor import ray_utils
401
- ray_utils.assert_ray_available()
402
-
389
+ def _get_executor_cls(
390
+ cls, engine_config: EngineConfig) -> Type[ExecutorAsyncBase]:
391
distributed_executor_backend = (
392
engine_config.parallel_config.distributed_executor_backend)
405
-
406
- if engine_config.device_config.device_type == "neuron":
393
+ if isinstance(distributed_executor_backend, type):
394
+ if not issubclass(distributed_executor_backend, ExecutorAsyncBase):
395
+ raise TypeError(
396
+ "distributed_executor_backend must be a subclass of "
397
+ f"ExecutorAsyncBase. Got {distributed_executor_backend}.")
398
+ if distributed_executor_backend.uses_ray: # type: ignore
399
+ initialize_ray_cluster(engine_config.parallel_config)
400
+ executor_class = distributed_executor_backend
401
+ elif engine_config.device_config.device_type == "neuron":
402
from vllm.executor.neuron_executor import NeuronExecutorAsync
403
executor_class = NeuronExecutorAsync
404
elif engine_config.device_config.device_type == "tpu":
@@ -442,9 +437,29 @@ class AsyncLLMEngine:
437
else:
438
from vllm.executor.gpu_executor import GPUExecutorAsync
439
executor_class = GPUExecutorAsync
440
+ return executor_class
441
+
442
+ @classmethod
443
+ def from_engine_args(
444
+ cls,
445
+ engine_args: AsyncEngineArgs,
446
+ start_engine_loop: bool = True,
447
+ usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
448
+ stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
449
+ ) -> "AsyncLLMEngine":
450
+ """Creates an async LLM engine from the engine arguments."""
451
+ # Create the engine configs.
452
+ engine_config = engine_args.create_engine_config()
453
+
454
+ if engine_args.engine_use_ray:
455
+ from vllm.executor import ray_utils
456
+ ray_utils.assert_ray_available()
457
+
458
+ executor_class = cls._get_executor_cls(engine_config)
459
+
460
# Create the async LLM engine.
461
engine = cls(
447
- distributed_executor_backend == "ray",
462
+ executor_class.uses_ray,
463
engine_args.engine_use_ray,
464
**engine_config.to_dict(),
465
executor_class=executor_class,
vllm/engine/llm_engine.py
+27
-13
@@ -7,9 +7,9 @@ from typing import Set, Type, TypeVar, Union
7
from transformers import PreTrainedTokenizer
8
9
import vllm.envs as envs
10
-from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
11
- LoRAConfig, ModelConfig, MultiModalConfig,
12
- ObservabilityConfig, ParallelConfig,
10
+from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
11
+ EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
12
+ MultiModalConfig, ObservabilityConfig, ParallelConfig,
13
PromptAdapterConfig, SchedulerConfig,
14
SpeculativeConfig)
15
from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler,
@@ -376,19 +376,20 @@ class LLMEngine:
376
self.model_executor.initialize_cache(num_gpu_blocks, num_cpu_blocks)
377
378
@classmethod
379
- def from_engine_args(
380
- cls,
381
- engine_args: EngineArgs,
382
- usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
383
- stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
384
- ) -> "LLMEngine":
385
- """Creates an LLM engine from the engine arguments."""
386
- # Create the engine configs.
387
- engine_config = engine_args.create_engine_config()
379
+ def _get_executor_cls(cls,
380
+ engine_config: EngineConfig) -> Type[ExecutorBase]:
381
distributed_executor_backend = (
382
engine_config.parallel_config.distributed_executor_backend)
383
# Initialize the cluster and specify the executor class.
391
- if engine_config.device_config.device_type == "neuron":
384
+ if isinstance(distributed_executor_backend, type):
385
+ if not issubclass(distributed_executor_backend, ExecutorBase):
386
+ raise TypeError(
387
+ "distributed_executor_backend must be a subclass of "
388
+ f"ExecutorBase. Got {distributed_executor_backend}.")
389
+ if distributed_executor_backend.uses_ray: # type: ignore
390
+ initialize_ray_cluster(engine_config.parallel_config)
391
+ executor_class = distributed_executor_backend
392
+ elif engine_config.device_config.device_type == "neuron":
393
from vllm.executor.neuron_executor import NeuronExecutor
394
executor_class = NeuronExecutor
395
elif engine_config.device_config.device_type == "tpu":
@@ -422,6 +423,19 @@ class LLMEngine:
423
else:
424
from vllm.executor.gpu_executor import GPUExecutor
425
executor_class = GPUExecutor
426
+ return executor_class
427
+
428
+ @classmethod
429
+ def from_engine_args(
430
+ cls,
431
+ engine_args: EngineArgs,
432
+ usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
433
+ stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
434
+ ) -> "LLMEngine":
435
+ """Creates an LLM engine from the engine arguments."""
436
+ # Create the engine configs.
437
+ engine_config = engine_args.create_engine_config()
438
+ executor_class = cls._get_executor_cls(engine_config)
439
# Create the LLM engine.
440
engine = cls(
441
**engine_config.to_dict(),
vllm/executor/cpu_executor.py
+2
@@ -17,6 +17,8 @@ logger = init_logger(__name__)
17
18
class CPUExecutor(ExecutorBase):
19
20
+ uses_ray: bool = False
21
+
22
def _init_executor(self) -> None:
23
assert self.device_config.device_type == "cpu"
24
assert self.lora_config is None, "cpu backend doesn't support LoRA"
vllm/executor/executor_base.py
+2
@@ -18,6 +18,8 @@ class ExecutorBase(ABC):
18
that can execute the model on multiple devices.
19
"""
20
21
+ uses_ray: bool # whether the executor uses Ray for orchestration.
22
+
23
def __init__(
24
self,
25
model_config: ModelConfig,
vllm/executor/gpu_executor.py
+2
@@ -23,6 +23,8 @@ def create_worker(worker_module_name, worker_class_name, **kwargs):
23
24
class GPUExecutor(ExecutorBase):
25
26
+ uses_ray: bool = False
27
+
28
def _init_executor(self) -> None:
29
"""Initialize the worker and load the model.
30
"""
vllm/executor/multiproc_gpu_executor.py
+2
@@ -25,6 +25,8 @@ logger = init_logger(__name__)
25
class MultiprocessingGPUExecutor(DistributedGPUExecutor):
26
"""Python multiprocessing-based multi-GPU executor"""
27
28
+ uses_ray: bool = False
29
+
30
def _init_executor(self) -> None:
31
# Create the parallel GPU workers.
32
world_size = self.parallel_config.world_size
vllm/executor/neuron_executor.py
+2
@@ -11,6 +11,8 @@ logger = init_logger(__name__)
11
12
class NeuronExecutor(ExecutorBase):
13
14
+ uses_ray: bool = False
15
+
16
def _init_executor(self) -> None:
17
assert (self.lora_config is
18
None), "LoRA is not supported for Neuron backend."
vllm/executor/openvino_executor.py
+2
@@ -18,6 +18,8 @@ logger = init_logger(__name__)
18
19
class OpenVINOExecutor(ExecutorBase):
20
21
+ uses_ray: bool = False
22
+
23
def _init_executor(self) -> None:
24
assert self.device_config.device_type == "openvino"
25
assert self.lora_config is None, "OpenVINO backend doesn't support LoRA"
vllm/executor/ray_gpu_executor.py
+21
-18
@@ -26,6 +26,8 @@ logger = init_logger(__name__)
26
27
class RayGPUExecutor(DistributedGPUExecutor):
28
29
+ uses_ray: bool = True
30
+
31
def _init_executor(self) -> None:
32
# If the env var is set, it uses the Ray's compiled DAG API
33
# which optimizes the control plane overhead.
@@ -47,7 +49,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
49
"VLLM_USE_RAY_SPMD_WORKER=1 requires "
50
"VLLM_USE_RAY_COMPILED_DAG=1")
51
50
- assert self.parallel_config.distributed_executor_backend == "ray"
52
+ assert self.uses_ray
53
placement_group = self.parallel_config.placement_group
54
55
# Disable Ray usage stats collection.
@@ -75,6 +77,20 @@ class RayGPUExecutor(DistributedGPUExecutor):
77
78
return ray_remote_kwargs
79
80
+ def _get_worker_wrapper_args(self) -> Dict[str, Any]:
81
+ if self.speculative_config is not None:
82
+ worker_module_name = "vllm.spec_decode.spec_decode_worker"
83
+ worker_class_name = "create_spec_worker"
84
+ else:
85
+ worker_module_name = "vllm.worker.worker"
86
+ worker_class_name = "Worker"
87
+
88
+ return dict(
89
+ worker_module_name=worker_module_name,
90
+ worker_class_name=worker_class_name,
91
+ trust_remote_code=self.model_config.trust_remote_code,
92
+ )
93
+
94
def _init_workers_ray(self, placement_group: "PlacementGroup",
95
**ray_remote_kwargs):
96
if (self.parallel_config.tensor_parallel_size == 1
@@ -97,6 +113,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
113
114
# Create the workers.
115
driver_ip = get_ip()
116
+ worker_wrapper_kwargs = self._get_worker_wrapper_args()
117
for bundle_id, bundle in enumerate(placement_group.bundle_specs):
118
if not bundle.get("GPU", 0):
119
continue
@@ -106,23 +123,12 @@ class RayGPUExecutor(DistributedGPUExecutor):
123
placement_group_bundle_index=bundle_id,
124
)
125
109
- if self.speculative_config is not None:
110
- worker_module_name = "vllm.spec_decode.spec_decode_worker"
111
- worker_class_name = "create_spec_worker"
112
- else:
113
- worker_module_name = "vllm.worker.worker"
114
- worker_class_name = "Worker"
115
-
126
worker = ray.remote(
127
num_cpus=0,
128
num_gpus=num_gpus,
129
scheduling_strategy=scheduling_strategy,
130
**ray_remote_kwargs,
121
- )(RayWorkerWrapper).remote(
122
- worker_module_name=worker_module_name,
123
- worker_class_name=worker_class_name,
124
- trust_remote_code=self.model_config.trust_remote_code,
125
- )
131
+ )(RayWorkerWrapper).remote(**worker_wrapper_kwargs)
132
133
if self.use_ray_spmd_worker:
134
self.workers.append(worker)
@@ -133,10 +139,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
139
# as the resource holder for the driver process.
140
self.driver_dummy_worker = worker
141
self.driver_worker = RayWorkerWrapper(
136
- worker_module_name=worker_module_name,
137
- worker_class_name=worker_class_name,
138
- trust_remote_code=self.model_config.trust_remote_code,
139
- )
142
+ **worker_wrapper_kwargs)
143
else:
144
# Else, added to the list of workers.
145
self.workers.append(worker)
@@ -378,7 +381,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
381
f"required, but found {current_version}")
382
383
from ray.dag import InputNode, MultiOutputNode
381
- assert self.parallel_config.distributed_executor_backend == "ray"
384
+ assert self.parallel_config.use_ray
385
386
# Right now, compiled DAG requires at least 1 arg. We send
387
# a dummy value for now. It will be fixed soon.
vllm/executor/ray_xpu_executor.py
+13
-11
@@ -35,6 +35,8 @@ USE_RAY_COMPILED_DAG = envs.VLLM_USE_RAY_COMPILED_DAG
35
36
class RayXPUExecutor(DistributedGPUExecutor):
37
38
+ uses_ray: bool = True
39
+
40
def __init__(
41
self,
42
model_config: ModelConfig,
@@ -107,6 +109,13 @@ class RayXPUExecutor(DistributedGPUExecutor):
109
110
return num_gpu_blocks, num_cpu_blocks
111
112
+ def _get_worker_wrapper_args(self) -> Dict[str, Any]:
113
+ return dict(
114
+ worker_module_name="vllm.worker.xpu_worker",
115
+ worker_class_name="XPUWorker",
116
+ trust_remote_code=self.model_config.trust_remote_code,
117
+ )
118
+
119
def _init_workers_ray(self, placement_group: "PlacementGroup",
120
**ray_remote_kwargs):
121
if self.parallel_config.tensor_parallel_size == 1:
@@ -124,6 +133,7 @@ class RayXPUExecutor(DistributedGPUExecutor):
133
134
# Create the workers.
135
driver_ip = get_ip()
136
+ worker_wrapper_kwargs = self._get_worker_wrapper_args()
137
for bundle_id, bundle in enumerate(placement_group.bundle_specs):
138
if not bundle.get("GPU", 0):
139
continue
@@ -137,22 +147,14 @@ class RayXPUExecutor(DistributedGPUExecutor):
147
num_gpus=num_gpus,
148
scheduling_strategy=scheduling_strategy,
149
**ray_remote_kwargs,
140
- )(RayWorkerWrapper).remote(
141
- worker_module_name="vllm.worker.xpu_worker",
142
- worker_class_name="XPUWorker",
143
- trust_remote_code=self.model_config.trust_remote_code,
144
- )
150
+ )(RayWorkerWrapper).remote(**worker_wrapper_kwargs)
151
152
worker_ip = ray.get(worker.get_node_ip.remote())
153
if worker_ip == driver_ip and self.driver_dummy_worker is None:
154
# If the worker is on the same node as the driver, we use it
155
# as the resource holder for the driver process.
156
self.driver_dummy_worker = worker
151
- self.driver_worker = RayWorkerWrapper(
152
- worker_module_name="vllm.worker.xpu_worker",
153
- worker_class_name="XPUWorker",
154
- trust_remote_code=self.model_config.trust_remote_code,
155
- )
157
+ self.driver_worker = RayWorkerWrapper(**worker_wrapper_kwargs)
158
else:
159
# Else, added to the list of workers.
160
self.workers.append(worker)
@@ -337,7 +339,7 @@ class RayXPUExecutor(DistributedGPUExecutor):
339
f"required, but found {current_version}")
340
341
from ray.dag import InputNode, MultiOutputNode
340
- assert self.parallel_config.distributed_executor_backend == "ray"
342
+ assert self.parallel_config.use_ray
343
344
# Right now, compiled DAG requires at least 1 arg. We send
345
# a dummy value for now. It will be fixed soon.
vllm/executor/tpu_executor.py
+2
@@ -14,6 +14,8 @@ logger = init_logger(__name__)
14
15
class TPUExecutor(ExecutorBase):
16
17
+ uses_ray: bool = False
18
+
19
def _init_executor(self) -> None:
20
assert not self.scheduler_config.chunked_prefill_enabled, (
21
"Chunked prefill is not yet supported for TPU backend")
vllm/executor/xpu_executor.py
+2
@@ -18,6 +18,8 @@ logger = init_logger(__name__)
18
19
class XPUExecutor(GPUExecutor):
20
21
+ uses_ray: bool = False
22
+
23
def __init__(
24
self,
25
model_config: ModelConfig,
vllm/transformers_utils/tokenizer_group/__init__.py
+9
-5
@@ -1,4 +1,4 @@
1
-from typing import Optional
1
+from typing import Optional, Type
2
3
from vllm.config import TokenizerPoolConfig
4
from vllm.executor.ray_utils import ray
@@ -16,18 +16,22 @@ else:
16
17
def get_tokenizer_group(tokenizer_pool_config: Optional[TokenizerPoolConfig],
18
**init_kwargs) -> BaseTokenizerGroup:
19
+ tokenizer_cls: Type[BaseTokenizerGroup]
20
if tokenizer_pool_config is None:
20
- return TokenizerGroup(**init_kwargs)
21
- if tokenizer_pool_config.pool_type == "ray":
21
+ tokenizer_cls = TokenizerGroup
22
+ elif isinstance(tokenizer_pool_config.pool_type, type) and issubclass(
23
+ tokenizer_pool_config.pool_type, BaseTokenizerGroup):
24
+ tokenizer_cls = tokenizer_pool_config.pool_type
25
+ elif tokenizer_pool_config.pool_type == "ray":
26
if RayTokenizerGroupPool is None:
27
raise ImportError(
28
"RayTokenizerGroupPool is not available. Please install "
29
"the ray package to use the Ray tokenizer group pool.")
26
- return RayTokenizerGroupPool.from_config(tokenizer_pool_config,
27
- **init_kwargs)
30
+ tokenizer_cls = RayTokenizerGroupPool
31
else:
32
raise ValueError(
33
f"Unknown pool type: {tokenizer_pool_config.pool_type}")
34
+ return tokenizer_cls.from_config(tokenizer_pool_config, **init_kwargs)
35
36
37
__all__ = ["get_tokenizer_group", "BaseTokenizerGroup"]
vllm/transformers_utils/tokenizer_group/base_tokenizer_group.py
+7
@@ -3,12 +3,19 @@ from typing import List, Optional
3
4
from transformers import PreTrainedTokenizer
5
6
+from vllm.config import TokenizerPoolConfig
7
from vllm.lora.request import LoRARequest
8
9
10
class BaseTokenizerGroup(ABC):
11
"""A group of tokenizers that can be used for LoRA adapters."""
12
13
+ @classmethod
14
+ @abstractmethod
15
+ def from_config(cls, tokenizer_pool_config: Optional[TokenizerPoolConfig],
16
+ **init_kwargs) -> "BaseTokenizerGroup":
17
+ pass
18
+
19
@abstractmethod
20
def ping(self) -> bool:
21
"""Check if the tokenizer group is alive."""
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
+3
-1
@@ -29,8 +29,10 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
29
_worker_cls = TokenizerGroup
30
31
@classmethod
32
- def from_config(cls, tokenizer_pool_config: TokenizerPoolConfig,
32
+ def from_config(cls, tokenizer_pool_config: Optional[TokenizerPoolConfig],
33
**init_kwargs) -> "RayTokenizerGroupPool":
34
+ if not tokenizer_pool_config:
35
+ raise ValueError("tokenizer_pool_config must not be None.")
36
ray_actor_options = (tokenizer_pool_config.extra_config or {
37
"num_cpus": 0
38
})
vllm/transformers_utils/tokenizer_group/tokenizer_group.py
+6
@@ -2,6 +2,7 @@ from typing import List, Optional
2
3
from transformers import PreTrainedTokenizer
4
5
+from vllm.config import TokenizerPoolConfig
6
from vllm.lora.request import LoRARequest
7
from vllm.transformers_utils.tokenizer import (get_lora_tokenizer,
8
get_lora_tokenizer_async,
@@ -24,6 +25,11 @@ class TokenizerGroup(BaseTokenizerGroup):
25
self.lora_tokenizers = LRUCache[PreTrainedTokenizer](
26
capacity=max_num_seqs) if enable_lora else None
27
28
+ @classmethod
29
+ def from_config(cls, tokenizer_pool_config: Optional[TokenizerPoolConfig],
30
+ **init_kwargs) -> "TokenizerGroup":
31
+ return cls(**init_kwargs)
32
+
33
def ping(self) -> bool:
34
"""Check if the tokenizer group is alive."""
35
return True
vllm/worker/worker_base.py
+19
-7
@@ -2,7 +2,7 @@ import dataclasses
2
import importlib
3
import os
4
from abc import ABC, abstractmethod
5
-from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union
5
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union
6
7
import torch
8
@@ -315,14 +315,23 @@ class WorkerWrapperBase:
315
We first instantiate the WorkerWrapper, which remembers the worker module
316
and class name. Then, when we call `update_environment_variables`, and the
317
real initialization happens in `init_worker`.
318
+
319
+ If worker_class_fn is specified, it will be executed to get the worker
320
+ class.
321
+ Otherwise, the worker class will be obtained by dynamically importing it
322
+ using worker_module_name and worker_class_name.
323
"""
324
320
- def __init__(self,
321
- worker_module_name: str,
322
- worker_class_name: str,
323
- trust_remote_code: bool = False) -> None:
325
+ def __init__(
326
+ self,
327
+ worker_module_name: str,
328
+ worker_class_name: str,
329
+ trust_remote_code: bool = False,
330
+ worker_class_fn: Optional[Callable[[],
331
+ Type[WorkerBase]]] = None) -> None:
332
self.worker_module_name = worker_module_name
333
self.worker_class_name = worker_class_name
334
+ self.worker_class_fn = worker_class_fn
335
self.worker: Optional[WorkerBase] = None
336
if trust_remote_code:
337
# note: lazy import to avoid importing torch before initializing
@@ -348,8 +357,11 @@ class WorkerWrapperBase:
357
# see https://github.com/NVIDIA/nccl/issues/1234
358
os.environ['NCCL_CUMEM_ENABLE'] = '0'
359
351
- mod = importlib.import_module(self.worker_module_name)
352
- worker_class = getattr(mod, self.worker_class_name)
360
+ if self.worker_class_fn:
361
+ worker_class = self.worker_class_fn()
362
+ else:
363
+ mod = importlib.import_module(self.worker_module_name)
364
+ worker_class = getattr(mod, self.worker_class_name)
365
366
self.worker = worker_class(*args, **kwargs)
367
assert self.worker is not None