@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7bd82002ae
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Core] Allow specifying custom Executor (#6557)

Antoni Baum committed Jul 19, 2024 at 18:25 UTC 7bd82002ae0e7ee8c4e5da0d43cfe0fd85372b4a
22 files changed +310 -92
tests/conftest.py
+4
@@ -564,6 +564,10 @@ def get_tokenizer_pool_config(tokenizer_group_type):
564 return TokenizerPoolConfig(pool_size=1,
565 pool_type="ray",
566 extra_config={})
567 + if isinstance(tokenizer_group_type, type):
568 + return TokenizerPoolConfig(pool_size=1,
569 + pool_type=tokenizer_group_type,
570 + extra_config={})
571 raise ValueError(f"Unknown tokenizer_group_type: {tokenizer_group_type}")
572
573
tests/engine/test_custom_executor.py new
+91
@@ -0,0 +1,91 @@
1 +import asyncio
2 +import os
3 +
4 +import pytest
5 +
6 +from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
7 +from vllm.engine.async_llm_engine import AsyncLLMEngine
8 +from vllm.engine.llm_engine import LLMEngine
9 +from vllm.executor.gpu_executor import GPUExecutor, GPUExecutorAsync
10 +from vllm.sampling_params import SamplingParams
11 +
12 +
13 +class Mock:
14 + ...
15 +
16 +
17 +class CustomGPUExecutor(GPUExecutor):
18 +
19 + def execute_model(self, *args, **kwargs):
20 + # Drop marker to show that this was ran
21 + with open(".marker", "w"):
22 + ...
23 + return super().execute_model(*args, **kwargs)
24 +
25 +
26 +class CustomGPUExecutorAsync(GPUExecutorAsync):
27 +
28 + async def execute_model_async(self, *args, **kwargs):
29 + with open(".marker", "w"):
30 + ...
31 + return await super().execute_model_async(*args, **kwargs)
32 +
33 +
34 +@pytest.mark.parametrize("model", ["facebook/opt-125m"])
35 +def test_custom_executor_type_checking(model):
36 + with pytest.raises(ValueError):
37 + engine_args = EngineArgs(model=model,
38 + distributed_executor_backend=Mock)
39 + LLMEngine.from_engine_args(engine_args)
40 + with pytest.raises(ValueError):
41 + engine_args = AsyncEngineArgs(model=model,
42 + distributed_executor_backend=Mock)
43 + AsyncLLMEngine.from_engine_args(engine_args)
44 + with pytest.raises(TypeError):
45 + engine_args = AsyncEngineArgs(
46 + model=model, distributed_executor_backend=CustomGPUExecutor)
47 + AsyncLLMEngine.from_engine_args(engine_args)
48 +
49 +
50 +@pytest.mark.parametrize("model", ["facebook/opt-125m"])
51 +def test_custom_executor(model, tmpdir):
52 + cwd = os.path.abspath(".")
53 + os.chdir(tmpdir)
54 + try:
55 + assert not os.path.exists(".marker")
56 +
57 + engine_args = EngineArgs(
58 + model=model, distributed_executor_backend=CustomGPUExecutor)
59 + engine = LLMEngine.from_engine_args(engine_args)
60 + sampling_params = SamplingParams(max_tokens=1)
61 +
62 + engine.add_request("0", "foo", sampling_params)
63 + engine.step()
64 +
65 + assert os.path.exists(".marker")
66 + finally:
67 + os.chdir(cwd)
68 +
69 +
70 +@pytest.mark.parametrize("model", ["facebook/opt-125m"])
71 +def test_custom_executor_async(model, tmpdir):
72 + cwd = os.path.abspath(".")
73 + os.chdir(tmpdir)
74 + try:
75 + assert not os.path.exists(".marker")
76 +
77 + engine_args = AsyncEngineArgs(
78 + model=model, distributed_executor_backend=CustomGPUExecutorAsync)
79 + engine = AsyncLLMEngine.from_engine_args(engine_args)
80 + sampling_params = SamplingParams(max_tokens=1)
81 +
82 + async def t():
83 + stream = await engine.add_request("0", "foo", sampling_params)
84 + async for x in stream:
85 + ...
86 +
87 + asyncio.run(t())
88 +
89 + assert os.path.exists(".marker")
90 + finally:
91 + os.chdir(cwd)
tests/tokenization/test_tokenizer_group.py
+17 -4
@@ -7,17 +7,28 @@ from unittest.mock import patch
7 import pytest
8 from transformers import AutoTokenizer, PreTrainedTokenizerBase
9
10 -from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
10 +from vllm.transformers_utils.tokenizer_group import (TokenizerGroup,
11 + get_tokenizer_group)
12 from vllm.transformers_utils.tokenizer_group.ray_tokenizer_group import (
13 RayTokenizerGroupPool)
13 -from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
14 - TokenizerGroup)
14
15 from ..conftest import get_tokenizer_pool_config
16
17
18 +class CustomTokenizerGroup(TokenizerGroup):
19 +
20 + def __init__(self, *args, **kwargs):
21 + super().__init__(*args, **kwargs)
22 + self._i = 0
23 +
24 + def encode(self, *args, **kwargs):
25 + self._i += 1
26 + return super().encode(*args, **kwargs)
27 +
28 +
29 @pytest.mark.asyncio
20 -@pytest.mark.parametrize("tokenizer_group_type", [None, "ray"])
30 +@pytest.mark.parametrize("tokenizer_group_type",
31 + [None, "ray", CustomTokenizerGroup])
32 async def test_tokenizer_group(tokenizer_group_type):
33 reference_tokenizer = AutoTokenizer.from_pretrained("gpt2")
34 tokenizer_group = get_tokenizer_group(
@@ -36,6 +47,8 @@ async def test_tokenizer_group(tokenizer_group_type):
47 PreTrainedTokenizerBase)
48 assert tokenizer_group.get_lora_tokenizer(
49 None) == await tokenizer_group.get_lora_tokenizer_async(None)
50 + if tokenizer_group_type is CustomTokenizerGroup:
51 + assert tokenizer_group._i > 0
52
53
54 @pytest.mark.asyncio
vllm/config.py
+28 -11
@@ -1,7 +1,7 @@
1 import enum
2 import json
3 from dataclasses import dataclass, field, fields
4 -from typing import TYPE_CHECKING, ClassVar, List, Optional, Tuple, Union
4 +from typing import TYPE_CHECKING, ClassVar, List, Optional, Tuple, Type, Union
5
6 import torch
7 from transformers import PretrainedConfig
@@ -18,7 +18,10 @@ from vllm.utils import (cuda_device_count_stateless, get_cpu_memory, is_cpu,
18 if TYPE_CHECKING:
19 from ray.util.placement_group import PlacementGroup
20
21 + from vllm.executor.executor_base import ExecutorBase
22 from vllm.model_executor.model_loader.loader import BaseModelLoader
23 + from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
24 + BaseTokenizerGroup)
25
26 logger = init_logger(__name__)
27
@@ -527,11 +530,12 @@ class TokenizerPoolConfig:
530 pool type.
531 """
532 pool_size: int
530 - pool_type: str
533 + pool_type: Union[str, Type["BaseTokenizerGroup"]]
534 extra_config: dict
535
536 def __post_init__(self):
534 - if self.pool_type not in ("ray", ):
537 + if self.pool_type not in ("ray", ) and not isinstance(
538 + self.pool_type, type):
539 raise ValueError(f"Unknown pool type: {self.pool_type}")
540 if not isinstance(self.extra_config, dict):
541 raise ValueError("extra_config must be a dictionary.")
@@ -661,7 +665,8 @@ class ParallelConfig:
665 tokenizer_pool_config: Optional[TokenizerPoolConfig] = None,
666 ray_workers_use_nsight: bool = False,
667 placement_group: Optional["PlacementGroup"] = None,
664 - distributed_executor_backend: Optional[str] = None,
668 + distributed_executor_backend: Optional[Union[
669 + str, Type["ExecutorBase"]]] = None,
670 ) -> None:
671 self.pipeline_parallel_size = pipeline_parallel_size
672 self.tensor_parallel_size = tensor_parallel_size
@@ -676,7 +681,7 @@ class ParallelConfig:
681 if worker_use_ray:
682 if self.distributed_executor_backend is None:
683 self.distributed_executor_backend = "ray"
679 - elif self.distributed_executor_backend != "ray":
684 + elif not self.use_ray:
685 raise ValueError(f"worker-use-ray can't be used with "
686 f"distributed executor backend "
687 f"'{self.distributed_executor_backend}'.")
@@ -711,12 +716,25 @@ class ParallelConfig:
716 self._verify_args()
717 self.rank = 0
718
719 + @property
720 + def use_ray(self) -> bool:
721 + return self.distributed_executor_backend == "ray" or (
722 + isinstance(self.distributed_executor_backend, type)
723 + and self.distributed_executor_backend.uses_ray)
724 +
725 def _verify_args(self) -> None:
715 - if self.distributed_executor_backend not in ("ray", "mp", None):
726 + # Lazy import to avoid circular import
727 + from vllm.executor.executor_base import ExecutorBase
728 +
729 + if self.distributed_executor_backend not in (
730 + "ray", "mp", None) and not (isinstance(
731 + self.distributed_executor_backend, type) and issubclass(
732 + self.distributed_executor_backend, ExecutorBase)):
733 raise ValueError(
717 - "Unrecognized distributed executor backend. Supported values "
718 - "are 'ray' or 'mp'.")
719 - if self.distributed_executor_backend == "ray":
734 + "Unrecognized distributed executor backend "
735 + f"{self.distributed_executor_backend}. Supported "
736 + "values are 'ray', 'mp' or custom ExecutorBase subclass.")
737 + if self.use_ray:
738 from vllm.executor import ray_utils
739 ray_utils.assert_ray_available()
740 if is_hip():
@@ -724,8 +742,7 @@ class ParallelConfig:
742 logger.info(
743 "Disabled the custom all-reduce kernel because it is not "
744 "supported on AMD GPUs.")
727 - if self.ray_workers_use_nsight and (
728 - not self.distributed_executor_backend == "ray"):
745 + if self.ray_workers_use_nsight and not self.use_ray:
746 raise ValueError("Unable to use nsight profiling unless workers "
747 "run with Ray.")
748
vllm/engine/arg_utils.py
+15 -3
@@ -2,16 +2,21 @@ import argparse
2 import dataclasses
3 import json
4 from dataclasses import dataclass
5 -from typing import List, Optional, Tuple, Union
5 +from typing import TYPE_CHECKING, List, Optional, Tuple, Type, Union
6
7 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
8 EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
9 MultiModalConfig, ObservabilityConfig, ParallelConfig,
10 PromptAdapterConfig, SchedulerConfig,
11 SpeculativeConfig, TokenizerPoolConfig)
12 +from vllm.executor.executor_base import ExecutorBase
13 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
14 from vllm.utils import FlexibleArgumentParser
15
16 +if TYPE_CHECKING:
17 + from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
18 + BaseTokenizerGroup)
19 +
20
21 def nullable_str(val: str):
22 if not val or val == "None":
@@ -36,7 +41,11 @@ class EngineArgs:
41 seed: int = 0
42 max_model_len: Optional[int] = None
43 worker_use_ray: bool = False
39 - distributed_executor_backend: Optional[str] = None
44 + # Note: Specifying a custom executor backend by passing a class
45 + # is intended for expert use only. The API may change without
46 + # notice.
47 + distributed_executor_backend: Optional[Union[str,
48 + Type[ExecutorBase]]] = None
49 pipeline_parallel_size: int = 1
50 tensor_parallel_size: int = 1
51 max_parallel_loading_workers: Optional[int] = None
@@ -62,7 +71,10 @@ class EngineArgs:
71 max_seq_len_to_capture: int = 8192
72 disable_custom_all_reduce: bool = False
73 tokenizer_pool_size: int = 0
65 - tokenizer_pool_type: str = "ray"
74 + # Note: Specifying a tokenizer pool by passing a class
75 + # is intended for expert use only. The API may change without
76 + # notice.
77 + tokenizer_pool_type: Union[str, Type["BaseTokenizerGroup"]] = "ray"
78 tokenizer_pool_extra_config: Optional[dict] = None
79 enable_lora: bool = False
80 max_loras: int = 1
vllm/engine/async_llm_engine.py
+34 -19
@@ -7,12 +7,13 @@ from typing import (AsyncIterator, Callable, Dict, Iterable, List, Optional,
7 from transformers import PreTrainedTokenizer
8
9 import vllm.envs as envs
10 -from vllm.config import DecodingConfig, ModelConfig
10 +from vllm.config import DecodingConfig, EngineConfig, ModelConfig
11 from vllm.core.scheduler import SchedulerOutputs
12 from vllm.engine.arg_utils import AsyncEngineArgs
13 from vllm.engine.async_timeout import asyncio_timeout
14 from vllm.engine.llm_engine import LLMEngine
15 from vllm.engine.metrics import StatLoggerBase
16 +from vllm.executor.executor_base import ExecutorAsyncBase
17 from vllm.executor.ray_utils import initialize_ray_cluster, ray
18 from vllm.inputs import LLMInputs, PromptInputs
19 from vllm.logger import init_logger
@@ -385,25 +386,19 @@ class AsyncLLMEngine:
386 self._request_tracker: RequestTracker
387
388 @classmethod
388 - def from_engine_args(
389 - cls,
390 - engine_args: AsyncEngineArgs,
391 - start_engine_loop: bool = True,
392 - usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
393 - stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
394 - ) -> "AsyncLLMEngine":
395 - """Creates an async LLM engine from the engine arguments."""
396 - # Create the engine configs.
397 - engine_config = engine_args.create_engine_config()
398 -
399 - if engine_args.engine_use_ray:
400 - from vllm.executor import ray_utils
401 - ray_utils.assert_ray_available()
402 -
389 + def _get_executor_cls(
390 + cls, engine_config: EngineConfig) -> Type[ExecutorAsyncBase]:
391 distributed_executor_backend = (
392 engine_config.parallel_config.distributed_executor_backend)
405 -
406 - if engine_config.device_config.device_type == "neuron":
393 + if isinstance(distributed_executor_backend, type):
394 + if not issubclass(distributed_executor_backend, ExecutorAsyncBase):
395 + raise TypeError(
396 + "distributed_executor_backend must be a subclass of "
397 + f"ExecutorAsyncBase. Got {distributed_executor_backend}.")
398 + if distributed_executor_backend.uses_ray: # type: ignore
399 + initialize_ray_cluster(engine_config.parallel_config)
400 + executor_class = distributed_executor_backend
401 + elif engine_config.device_config.device_type == "neuron":
402 from vllm.executor.neuron_executor import NeuronExecutorAsync
403 executor_class = NeuronExecutorAsync
404 elif engine_config.device_config.device_type == "tpu":
@@ -442,9 +437,29 @@ class AsyncLLMEngine:
437 else:
438 from vllm.executor.gpu_executor import GPUExecutorAsync
439 executor_class = GPUExecutorAsync
440 + return executor_class
441 +
442 + @classmethod
443 + def from_engine_args(
444 + cls,
445 + engine_args: AsyncEngineArgs,
446 + start_engine_loop: bool = True,
447 + usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
448 + stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
449 + ) -> "AsyncLLMEngine":
450 + """Creates an async LLM engine from the engine arguments."""
451 + # Create the engine configs.
452 + engine_config = engine_args.create_engine_config()
453 +
454 + if engine_args.engine_use_ray:
455 + from vllm.executor import ray_utils
456 + ray_utils.assert_ray_available()
457 +
458 + executor_class = cls._get_executor_cls(engine_config)
459 +
460 # Create the async LLM engine.
461 engine = cls(
447 - distributed_executor_backend == "ray",
462 + executor_class.uses_ray,
463 engine_args.engine_use_ray,
464 **engine_config.to_dict(),
465 executor_class=executor_class,
vllm/engine/llm_engine.py
+27 -13
@@ -7,9 +7,9 @@ from typing import Set, Type, TypeVar, Union
7 from transformers import PreTrainedTokenizer
8
9 import vllm.envs as envs
10 -from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
11 - LoRAConfig, ModelConfig, MultiModalConfig,
12 - ObservabilityConfig, ParallelConfig,
10 +from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
11 + EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
12 + MultiModalConfig, ObservabilityConfig, ParallelConfig,
13 PromptAdapterConfig, SchedulerConfig,
14 SpeculativeConfig)
15 from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler,
@@ -376,19 +376,20 @@ class LLMEngine:
376 self.model_executor.initialize_cache(num_gpu_blocks, num_cpu_blocks)
377
378 @classmethod
379 - def from_engine_args(
380 - cls,
381 - engine_args: EngineArgs,
382 - usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
383 - stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
384 - ) -> "LLMEngine":
385 - """Creates an LLM engine from the engine arguments."""
386 - # Create the engine configs.
387 - engine_config = engine_args.create_engine_config()
379 + def _get_executor_cls(cls,
380 + engine_config: EngineConfig) -> Type[ExecutorBase]:
381 distributed_executor_backend = (
382 engine_config.parallel_config.distributed_executor_backend)
383 # Initialize the cluster and specify the executor class.
391 - if engine_config.device_config.device_type == "neuron":
384 + if isinstance(distributed_executor_backend, type):
385 + if not issubclass(distributed_executor_backend, ExecutorBase):
386 + raise TypeError(
387 + "distributed_executor_backend must be a subclass of "
388 + f"ExecutorBase. Got {distributed_executor_backend}.")
389 + if distributed_executor_backend.uses_ray: # type: ignore
390 + initialize_ray_cluster(engine_config.parallel_config)
391 + executor_class = distributed_executor_backend
392 + elif engine_config.device_config.device_type == "neuron":
393 from vllm.executor.neuron_executor import NeuronExecutor
394 executor_class = NeuronExecutor
395 elif engine_config.device_config.device_type == "tpu":
@@ -422,6 +423,19 @@ class LLMEngine:
423 else:
424 from vllm.executor.gpu_executor import GPUExecutor
425 executor_class = GPUExecutor
426 + return executor_class
427 +
428 + @classmethod
429 + def from_engine_args(
430 + cls,
431 + engine_args: EngineArgs,
432 + usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
433 + stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
434 + ) -> "LLMEngine":
435 + """Creates an LLM engine from the engine arguments."""
436 + # Create the engine configs.
437 + engine_config = engine_args.create_engine_config()
438 + executor_class = cls._get_executor_cls(engine_config)
439 # Create the LLM engine.
440 engine = cls(
441 **engine_config.to_dict(),
vllm/executor/cpu_executor.py
+2
@@ -17,6 +17,8 @@ logger = init_logger(__name__)
17
18 class CPUExecutor(ExecutorBase):
19
20 + uses_ray: bool = False
21 +
22 def _init_executor(self) -> None:
23 assert self.device_config.device_type == "cpu"
24 assert self.lora_config is None, "cpu backend doesn't support LoRA"
vllm/executor/executor_base.py
+2
@@ -18,6 +18,8 @@ class ExecutorBase(ABC):
18 that can execute the model on multiple devices.
19 """
20
21 + uses_ray: bool # whether the executor uses Ray for orchestration.
22 +
23 def __init__(
24 self,
25 model_config: ModelConfig,
vllm/executor/gpu_executor.py
+2
@@ -23,6 +23,8 @@ def create_worker(worker_module_name, worker_class_name, **kwargs):
23
24 class GPUExecutor(ExecutorBase):
25
26 + uses_ray: bool = False
27 +
28 def _init_executor(self) -> None:
29 """Initialize the worker and load the model.
30 """
vllm/executor/multiproc_gpu_executor.py
+2
@@ -25,6 +25,8 @@ logger = init_logger(__name__)
25 class MultiprocessingGPUExecutor(DistributedGPUExecutor):
26 """Python multiprocessing-based multi-GPU executor"""
27
28 + uses_ray: bool = False
29 +
30 def _init_executor(self) -> None:
31 # Create the parallel GPU workers.
32 world_size = self.parallel_config.world_size
vllm/executor/neuron_executor.py
+2
@@ -11,6 +11,8 @@ logger = init_logger(__name__)
11
12 class NeuronExecutor(ExecutorBase):
13
14 + uses_ray: bool = False
15 +
16 def _init_executor(self) -> None:
17 assert (self.lora_config is
18 None), "LoRA is not supported for Neuron backend."
vllm/executor/openvino_executor.py
+2
@@ -18,6 +18,8 @@ logger = init_logger(__name__)
18
19 class OpenVINOExecutor(ExecutorBase):
20
21 + uses_ray: bool = False
22 +
23 def _init_executor(self) -> None:
24 assert self.device_config.device_type == "openvino"
25 assert self.lora_config is None, "OpenVINO backend doesn't support LoRA"
vllm/executor/ray_gpu_executor.py
+21 -18
@@ -26,6 +26,8 @@ logger = init_logger(__name__)
26
27 class RayGPUExecutor(DistributedGPUExecutor):
28
29 + uses_ray: bool = True
30 +
31 def _init_executor(self) -> None:
32 # If the env var is set, it uses the Ray's compiled DAG API
33 # which optimizes the control plane overhead.
@@ -47,7 +49,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
49 "VLLM_USE_RAY_SPMD_WORKER=1 requires "
50 "VLLM_USE_RAY_COMPILED_DAG=1")
51
50 - assert self.parallel_config.distributed_executor_backend == "ray"
52 + assert self.uses_ray
53 placement_group = self.parallel_config.placement_group
54
55 # Disable Ray usage stats collection.
@@ -75,6 +77,20 @@ class RayGPUExecutor(DistributedGPUExecutor):
77
78 return ray_remote_kwargs
79
80 + def _get_worker_wrapper_args(self) -> Dict[str, Any]:
81 + if self.speculative_config is not None:
82 + worker_module_name = "vllm.spec_decode.spec_decode_worker"
83 + worker_class_name = "create_spec_worker"
84 + else:
85 + worker_module_name = "vllm.worker.worker"
86 + worker_class_name = "Worker"
87 +
88 + return dict(
89 + worker_module_name=worker_module_name,
90 + worker_class_name=worker_class_name,
91 + trust_remote_code=self.model_config.trust_remote_code,
92 + )
93 +
94 def _init_workers_ray(self, placement_group: "PlacementGroup",
95 **ray_remote_kwargs):
96 if (self.parallel_config.tensor_parallel_size == 1
@@ -97,6 +113,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
113
114 # Create the workers.
115 driver_ip = get_ip()
116 + worker_wrapper_kwargs = self._get_worker_wrapper_args()
117 for bundle_id, bundle in enumerate(placement_group.bundle_specs):
118 if not bundle.get("GPU", 0):
119 continue
@@ -106,23 +123,12 @@ class RayGPUExecutor(DistributedGPUExecutor):
123 placement_group_bundle_index=bundle_id,
124 )
125
109 - if self.speculative_config is not None:
110 - worker_module_name = "vllm.spec_decode.spec_decode_worker"
111 - worker_class_name = "create_spec_worker"
112 - else:
113 - worker_module_name = "vllm.worker.worker"
114 - worker_class_name = "Worker"
115 -
126 worker = ray.remote(
127 num_cpus=0,
128 num_gpus=num_gpus,
129 scheduling_strategy=scheduling_strategy,
130 **ray_remote_kwargs,
121 - )(RayWorkerWrapper).remote(
122 - worker_module_name=worker_module_name,
123 - worker_class_name=worker_class_name,
124 - trust_remote_code=self.model_config.trust_remote_code,
125 - )
131 + )(RayWorkerWrapper).remote(**worker_wrapper_kwargs)
132
133 if self.use_ray_spmd_worker:
134 self.workers.append(worker)
@@ -133,10 +139,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
139 # as the resource holder for the driver process.
140 self.driver_dummy_worker = worker
141 self.driver_worker = RayWorkerWrapper(
136 - worker_module_name=worker_module_name,
137 - worker_class_name=worker_class_name,
138 - trust_remote_code=self.model_config.trust_remote_code,
139 - )
142 + **worker_wrapper_kwargs)
143 else:
144 # Else, added to the list of workers.
145 self.workers.append(worker)
@@ -378,7 +381,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
381 f"required, but found {current_version}")
382
383 from ray.dag import InputNode, MultiOutputNode
381 - assert self.parallel_config.distributed_executor_backend == "ray"
384 + assert self.parallel_config.use_ray
385
386 # Right now, compiled DAG requires at least 1 arg. We send
387 # a dummy value for now. It will be fixed soon.
vllm/executor/ray_xpu_executor.py
+13 -11
@@ -35,6 +35,8 @@ USE_RAY_COMPILED_DAG = envs.VLLM_USE_RAY_COMPILED_DAG
35
36 class RayXPUExecutor(DistributedGPUExecutor):
37
38 + uses_ray: bool = True
39 +
40 def __init__(
41 self,
42 model_config: ModelConfig,
@@ -107,6 +109,13 @@ class RayXPUExecutor(DistributedGPUExecutor):
109
110 return num_gpu_blocks, num_cpu_blocks
111
112 + def _get_worker_wrapper_args(self) -> Dict[str, Any]:
113 + return dict(
114 + worker_module_name="vllm.worker.xpu_worker",
115 + worker_class_name="XPUWorker",
116 + trust_remote_code=self.model_config.trust_remote_code,
117 + )
118 +
119 def _init_workers_ray(self, placement_group: "PlacementGroup",
120 **ray_remote_kwargs):
121 if self.parallel_config.tensor_parallel_size == 1:
@@ -124,6 +133,7 @@ class RayXPUExecutor(DistributedGPUExecutor):
133
134 # Create the workers.
135 driver_ip = get_ip()
136 + worker_wrapper_kwargs = self._get_worker_wrapper_args()
137 for bundle_id, bundle in enumerate(placement_group.bundle_specs):
138 if not bundle.get("GPU", 0):
139 continue
@@ -137,22 +147,14 @@ class RayXPUExecutor(DistributedGPUExecutor):
147 num_gpus=num_gpus,
148 scheduling_strategy=scheduling_strategy,
149 **ray_remote_kwargs,
140 - )(RayWorkerWrapper).remote(
141 - worker_module_name="vllm.worker.xpu_worker",
142 - worker_class_name="XPUWorker",
143 - trust_remote_code=self.model_config.trust_remote_code,
144 - )
150 + )(RayWorkerWrapper).remote(**worker_wrapper_kwargs)
151
152 worker_ip = ray.get(worker.get_node_ip.remote())
153 if worker_ip == driver_ip and self.driver_dummy_worker is None:
154 # If the worker is on the same node as the driver, we use it
155 # as the resource holder for the driver process.
156 self.driver_dummy_worker = worker
151 - self.driver_worker = RayWorkerWrapper(
152 - worker_module_name="vllm.worker.xpu_worker",
153 - worker_class_name="XPUWorker",
154 - trust_remote_code=self.model_config.trust_remote_code,
155 - )
157 + self.driver_worker = RayWorkerWrapper(**worker_wrapper_kwargs)
158 else:
159 # Else, added to the list of workers.
160 self.workers.append(worker)
@@ -337,7 +339,7 @@ class RayXPUExecutor(DistributedGPUExecutor):
339 f"required, but found {current_version}")
340
341 from ray.dag import InputNode, MultiOutputNode
340 - assert self.parallel_config.distributed_executor_backend == "ray"
342 + assert self.parallel_config.use_ray
343
344 # Right now, compiled DAG requires at least 1 arg. We send
345 # a dummy value for now. It will be fixed soon.
vllm/executor/tpu_executor.py
+2
@@ -14,6 +14,8 @@ logger = init_logger(__name__)
14
15 class TPUExecutor(ExecutorBase):
16
17 + uses_ray: bool = False
18 +
19 def _init_executor(self) -> None:
20 assert not self.scheduler_config.chunked_prefill_enabled, (
21 "Chunked prefill is not yet supported for TPU backend")
vllm/executor/xpu_executor.py
+2
@@ -18,6 +18,8 @@ logger = init_logger(__name__)
18
19 class XPUExecutor(GPUExecutor):
20
21 + uses_ray: bool = False
22 +
23 def __init__(
24 self,
25 model_config: ModelConfig,
vllm/transformers_utils/tokenizer_group/__init__.py
+9 -5
@@ -1,4 +1,4 @@
1 -from typing import Optional
1 +from typing import Optional, Type
2
3 from vllm.config import TokenizerPoolConfig
4 from vllm.executor.ray_utils import ray
@@ -16,18 +16,22 @@ else:
16
17 def get_tokenizer_group(tokenizer_pool_config: Optional[TokenizerPoolConfig],
18 **init_kwargs) -> BaseTokenizerGroup:
19 + tokenizer_cls: Type[BaseTokenizerGroup]
20 if tokenizer_pool_config is None:
20 - return TokenizerGroup(**init_kwargs)
21 - if tokenizer_pool_config.pool_type == "ray":
21 + tokenizer_cls = TokenizerGroup
22 + elif isinstance(tokenizer_pool_config.pool_type, type) and issubclass(
23 + tokenizer_pool_config.pool_type, BaseTokenizerGroup):
24 + tokenizer_cls = tokenizer_pool_config.pool_type
25 + elif tokenizer_pool_config.pool_type == "ray":
26 if RayTokenizerGroupPool is None:
27 raise ImportError(
28 "RayTokenizerGroupPool is not available. Please install "
29 "the ray package to use the Ray tokenizer group pool.")
26 - return RayTokenizerGroupPool.from_config(tokenizer_pool_config,
27 - **init_kwargs)
30 + tokenizer_cls = RayTokenizerGroupPool
31 else:
32 raise ValueError(
33 f"Unknown pool type: {tokenizer_pool_config.pool_type}")
34 + return tokenizer_cls.from_config(tokenizer_pool_config, **init_kwargs)
35
36
37 __all__ = ["get_tokenizer_group", "BaseTokenizerGroup"]
vllm/transformers_utils/tokenizer_group/base_tokenizer_group.py
+7
@@ -3,12 +3,19 @@ from typing import List, Optional
3
4 from transformers import PreTrainedTokenizer
5
6 +from vllm.config import TokenizerPoolConfig
7 from vllm.lora.request import LoRARequest
8
9
10 class BaseTokenizerGroup(ABC):
11 """A group of tokenizers that can be used for LoRA adapters."""
12
13 + @classmethod
14 + @abstractmethod
15 + def from_config(cls, tokenizer_pool_config: Optional[TokenizerPoolConfig],
16 + **init_kwargs) -> "BaseTokenizerGroup":
17 + pass
18 +
19 @abstractmethod
20 def ping(self) -> bool:
21 """Check if the tokenizer group is alive."""
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
+3 -1
@@ -29,8 +29,10 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
29 _worker_cls = TokenizerGroup
30
31 @classmethod
32 - def from_config(cls, tokenizer_pool_config: TokenizerPoolConfig,
32 + def from_config(cls, tokenizer_pool_config: Optional[TokenizerPoolConfig],
33 **init_kwargs) -> "RayTokenizerGroupPool":
34 + if not tokenizer_pool_config:
35 + raise ValueError("tokenizer_pool_config must not be None.")
36 ray_actor_options = (tokenizer_pool_config.extra_config or {
37 "num_cpus": 0
38 })
vllm/transformers_utils/tokenizer_group/tokenizer_group.py
+6
@@ -2,6 +2,7 @@ from typing import List, Optional
2
3 from transformers import PreTrainedTokenizer
4
5 +from vllm.config import TokenizerPoolConfig
6 from vllm.lora.request import LoRARequest
7 from vllm.transformers_utils.tokenizer import (get_lora_tokenizer,
8 get_lora_tokenizer_async,
@@ -24,6 +25,11 @@ class TokenizerGroup(BaseTokenizerGroup):
25 self.lora_tokenizers = LRUCache[PreTrainedTokenizer](
26 capacity=max_num_seqs) if enable_lora else None
27
28 + @classmethod
29 + def from_config(cls, tokenizer_pool_config: Optional[TokenizerPoolConfig],
30 + **init_kwargs) -> "TokenizerGroup":
31 + return cls(**init_kwargs)
32 +
33 def ping(self) -> bool:
34 """Check if the tokenizer group is alive."""
35 return True
vllm/worker/worker_base.py
+19 -7
@@ -2,7 +2,7 @@ import dataclasses
2 import importlib
3 import os
4 from abc import ABC, abstractmethod
5 -from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union
5 +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union
6
7 import torch
8
@@ -315,14 +315,23 @@ class WorkerWrapperBase:
315 We first instantiate the WorkerWrapper, which remembers the worker module
316 and class name. Then, when we call `update_environment_variables`, and the
317 real initialization happens in `init_worker`.
318 +
319 + If worker_class_fn is specified, it will be executed to get the worker
320 + class.
321 + Otherwise, the worker class will be obtained by dynamically importing it
322 + using worker_module_name and worker_class_name.
323 """
324
320 - def __init__(self,
321 - worker_module_name: str,
322 - worker_class_name: str,
323 - trust_remote_code: bool = False) -> None:
325 + def __init__(
326 + self,
327 + worker_module_name: str,
328 + worker_class_name: str,
329 + trust_remote_code: bool = False,
330 + worker_class_fn: Optional[Callable[[],
331 + Type[WorkerBase]]] = None) -> None:
332 self.worker_module_name = worker_module_name
333 self.worker_class_name = worker_class_name
334 + self.worker_class_fn = worker_class_fn
335 self.worker: Optional[WorkerBase] = None
336 if trust_remote_code:
337 # note: lazy import to avoid importing torch before initializing
@@ -348,8 +357,11 @@ class WorkerWrapperBase:
357 # see https://github.com/NVIDIA/nccl/issues/1234
358 os.environ['NCCL_CUMEM_ENABLE'] = '0'
359
351 - mod = importlib.import_module(self.worker_module_name)
352 - worker_class = getattr(mod, self.worker_class_name)
360 + if self.worker_class_fn:
361 + worker_class = self.worker_class_fn()
362 + else:
363 + mod = importlib.import_module(self.worker_module_name)
364 + worker_class = getattr(mod, self.worker_class_name)
365
366 self.worker = worker_class(*args, **kwargs)
367 assert self.worker is not None