@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 66a9e713a7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Core] Pipe `worker_class_fn` argument in Executor (#7707)

Antoni Baum committed Aug 20, 2024 at 17:37 UTC 66a9e713a70d71070c77342b85e020ce536f13c0
3 files changed +26 -14
vllm/executor/gpu_executor.py
+17 -9
@@ -1,4 +1,4 @@
1 -from typing import Any, Dict, List, Optional, Set, Tuple, Union
1 +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union
2
3 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4 from vllm.logger import init_logger
@@ -7,15 +7,18 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
7 from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
8 from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
9 make_async)
10 -from vllm.worker.worker_base import WorkerWrapperBase
10 +from vllm.worker.worker_base import WorkerBase, WorkerWrapperBase
11
12 logger = init_logger(__name__)
13
14
15 -def create_worker(worker_module_name, worker_class_name, **kwargs):
15 +def create_worker(worker_module_name: str, worker_class_name: str,
16 + worker_class_fn: Optional[Callable[[], Type[WorkerBase]]],
17 + **kwargs):
18 wrapper = WorkerWrapperBase(
19 worker_module_name=worker_module_name,
20 worker_class_name=worker_class_name,
21 + worker_class_fn=worker_class_fn,
22 )
23 wrapper.init_worker(**kwargs)
24 return wrapper.worker
@@ -62,7 +65,9 @@ class GPUExecutor(ExecutorBase):
65 observability_config=self.observability_config,
66 )
67
65 - def _get_worker_module_and_class(self) -> Tuple[str, str]:
68 + def _get_worker_module_and_class(
69 + self) -> Tuple[str, str, Optional[Callable[[], Type[WorkerBase]]]]:
70 + worker_class_fn = None
71 if self.scheduler_config.is_multi_step:
72 worker_module_name = "vllm.worker.multi_step_worker"
73 worker_class_name = "MultiStepWorker"
@@ -72,7 +77,7 @@ class GPUExecutor(ExecutorBase):
77 else:
78 worker_module_name = "vllm.worker.worker"
79 worker_class_name = "Worker"
75 - return (worker_module_name, worker_class_name)
80 + return (worker_module_name, worker_class_name, worker_class_fn)
81
82 def _get_create_worker_kwargs(
83 self,
@@ -82,10 +87,13 @@ class GPUExecutor(ExecutorBase):
87 worker_kwargs = self._get_worker_kwargs(local_rank, rank,
88 distributed_init_method)
89
85 - (worker_module_name,
86 - worker_class_name) = self._get_worker_module_and_class()
87 - worker_kwargs.update(worker_module_name=worker_module_name,
88 - worker_class_name=worker_class_name)
90 + (worker_module_name, worker_class_name,
91 + worker_class_fn) = self._get_worker_module_and_class()
92 + worker_kwargs.update(
93 + worker_module_name=worker_module_name,
94 + worker_class_name=worker_class_name,
95 + worker_class_fn=worker_class_fn,
96 + )
97
98 return worker_kwargs
99
vllm/executor/ray_gpu_executor.py
+3 -2
@@ -91,12 +91,13 @@ class RayGPUExecutor(DistributedGPUExecutor):
91 return ray_remote_kwargs
92
93 def _get_worker_wrapper_args(self) -> Dict[str, Any]:
94 - (worker_module_name,
95 - worker_class_name) = self._get_worker_module_and_class()
94 + (worker_module_name, worker_class_name,
95 + worker_class_fn) = self._get_worker_module_and_class()
96
97 return dict(
98 worker_module_name=worker_module_name,
99 worker_class_name=worker_class_name,
100 + worker_class_fn=worker_class_fn,
101 trust_remote_code=self.model_config.trust_remote_code,
102 )
103
vllm/executor/xpu_executor.py
+6 -3
@@ -1,4 +1,4 @@
1 -from typing import List, Optional, Tuple, Union
1 +from typing import Callable, List, Optional, Tuple, Type, Union
2
3 import torch
4
@@ -11,6 +11,7 @@ from vllm.executor.gpu_executor import GPUExecutor
11 from vllm.logger import init_logger
12 from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
13 from vllm.utils import make_async
14 +from vllm.worker.worker_base import WorkerBase
15
16 logger = init_logger(__name__)
17
@@ -52,14 +53,16 @@ class XPUExecutor(GPUExecutor):
53 # Instantiate the worker and load the model to GPU.
54 self._init_executor()
55
55 - def _get_worker_module_and_class(self) -> Tuple[str, str]:
56 + def _get_worker_module_and_class(
57 + self) -> Tuple[str, str, Optional[Callable[[], Type[WorkerBase]]]]:
58 + worker_class_fn = None
59 if self.speculative_config is not None:
60 raise NotImplementedError(
61 "XPU does not support speculative decoding")
62 else:
63 worker_module_name = "vllm.worker.xpu_worker"
64 worker_class_name = "XPUWorker"
62 - return (worker_module_name, worker_class_name)
65 + return (worker_module_name, worker_class_name, worker_class_fn)
66
67 def execute_model(
68 self, execute_model_req: ExecuteModelRequest