A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Core] Pipe `worker_class_fn` argument in Executor (#7707)
Antoni Baum committed
Aug 20, 2024 at 17:37 UTC
66a9e713a70d71070c77342b85e020ce536f13c0
3 files changed
+26
-14
vllm/executor/gpu_executor.py
+17
-9
@@ -1,4 +1,4 @@
1
-from typing import Any, Dict, List, Optional, Set, Tuple, Union
1
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union
2
3
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4
from vllm.logger import init_logger
@@ -7,15 +7,18 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
7
from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
8
from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
9
make_async)
10
-from vllm.worker.worker_base import WorkerWrapperBase
10
+from vllm.worker.worker_base import WorkerBase, WorkerWrapperBase
11
12
logger = init_logger(__name__)
13
14
15
-def create_worker(worker_module_name, worker_class_name, **kwargs):
15
+def create_worker(worker_module_name: str, worker_class_name: str,
16
+ worker_class_fn: Optional[Callable[[], Type[WorkerBase]]],
17
+ **kwargs):
18
wrapper = WorkerWrapperBase(
19
worker_module_name=worker_module_name,
20
worker_class_name=worker_class_name,
21
+ worker_class_fn=worker_class_fn,
22
)
23
wrapper.init_worker(**kwargs)
24
return wrapper.worker
@@ -62,7 +65,9 @@ class GPUExecutor(ExecutorBase):
65
observability_config=self.observability_config,
66
)
67
65
- def _get_worker_module_and_class(self) -> Tuple[str, str]:
68
+ def _get_worker_module_and_class(
69
+ self) -> Tuple[str, str, Optional[Callable[[], Type[WorkerBase]]]]:
70
+ worker_class_fn = None
71
if self.scheduler_config.is_multi_step:
72
worker_module_name = "vllm.worker.multi_step_worker"
73
worker_class_name = "MultiStepWorker"
@@ -72,7 +77,7 @@ class GPUExecutor(ExecutorBase):
77
else:
78
worker_module_name = "vllm.worker.worker"
79
worker_class_name = "Worker"
75
- return (worker_module_name, worker_class_name)
80
+ return (worker_module_name, worker_class_name, worker_class_fn)
81
82
def _get_create_worker_kwargs(
83
self,
@@ -82,10 +87,13 @@ class GPUExecutor(ExecutorBase):
87
worker_kwargs = self._get_worker_kwargs(local_rank, rank,
88
distributed_init_method)
89
85
- (worker_module_name,
86
- worker_class_name) = self._get_worker_module_and_class()
87
- worker_kwargs.update(worker_module_name=worker_module_name,
88
- worker_class_name=worker_class_name)
90
+ (worker_module_name, worker_class_name,
91
+ worker_class_fn) = self._get_worker_module_and_class()
92
+ worker_kwargs.update(
93
+ worker_module_name=worker_module_name,
94
+ worker_class_name=worker_class_name,
95
+ worker_class_fn=worker_class_fn,
96
+ )
97
98
return worker_kwargs
99
vllm/executor/ray_gpu_executor.py
+3
-2
@@ -91,12 +91,13 @@ class RayGPUExecutor(DistributedGPUExecutor):
91
return ray_remote_kwargs
92
93
def _get_worker_wrapper_args(self) -> Dict[str, Any]:
94
- (worker_module_name,
95
- worker_class_name) = self._get_worker_module_and_class()
94
+ (worker_module_name, worker_class_name,
95
+ worker_class_fn) = self._get_worker_module_and_class()
96
97
return dict(
98
worker_module_name=worker_module_name,
99
worker_class_name=worker_class_name,
100
+ worker_class_fn=worker_class_fn,
101
trust_remote_code=self.model_config.trust_remote_code,
102
)
103
vllm/executor/xpu_executor.py
+6
-3
@@ -1,4 +1,4 @@
1
-from typing import List, Optional, Tuple, Union
1
+from typing import Callable, List, Optional, Tuple, Type, Union
2
3
import torch
4
@@ -11,6 +11,7 @@ from vllm.executor.gpu_executor import GPUExecutor
11
from vllm.logger import init_logger
12
from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
13
from vllm.utils import make_async
14
+from vllm.worker.worker_base import WorkerBase
15
16
logger = init_logger(__name__)
17
@@ -52,14 +53,16 @@ class XPUExecutor(GPUExecutor):
53
# Instantiate the worker and load the model to GPU.
54
self._init_executor()
55
55
- def _get_worker_module_and_class(self) -> Tuple[str, str]:
56
+ def _get_worker_module_and_class(
57
+ self) -> Tuple[str, str, Optional[Callable[[], Type[WorkerBase]]]]:
58
+ worker_class_fn = None
59
if self.speculative_config is not None:
60
raise NotImplementedError(
61
"XPU does not support speculative decoding")
62
else:
63
worker_module_name = "vllm.worker.xpu_worker"
64
worker_class_name = "XPUWorker"
62
- return (worker_module_name, worker_class_name)
65
+ return (worker_module_name, worker_class_name, worker_class_fn)
66
67
def execute_model(
68
self, execute_model_req: ExecuteModelRequest