A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Core] Refactor executor classes for easier inheritance (#7673)
[Core] Refactor executor classes to make it easier to inherit GPUExecutor (#7673)
Kunshang Ji committed
Aug 20, 2024 at 15:56 UTC
b6f99a6ffe4c7471b32932a0d163fe0e7bbead5b
2 files changed
+27
-21
vllm/executor/gpu_executor.py
+16
-11
@@ -62,6 +62,18 @@ class GPUExecutor(ExecutorBase):
62
observability_config=self.observability_config,
63
)
64
65
+ def _get_worker_module_and_class(self) -> Tuple[str, str]:
66
+ if self.scheduler_config.is_multi_step:
67
+ worker_module_name = "vllm.worker.multi_step_worker"
68
+ worker_class_name = "MultiStepWorker"
69
+ elif self.speculative_config:
70
+ worker_module_name = "vllm.spec_decode.spec_decode_worker"
71
+ worker_class_name = "create_spec_worker"
72
+ else:
73
+ worker_module_name = "vllm.worker.worker"
74
+ worker_class_name = "Worker"
75
+ return (worker_module_name, worker_class_name)
76
+
77
def _get_create_worker_kwargs(
78
self,
79
local_rank: int = 0,
@@ -70,17 +82,10 @@ class GPUExecutor(ExecutorBase):
82
worker_kwargs = self._get_worker_kwargs(local_rank, rank,
83
distributed_init_method)
84
73
- if self.scheduler_config.is_multi_step:
74
- worker_kwargs.update(
75
- worker_module_name="vllm.worker.multi_step_worker",
76
- worker_class_name="MultiStepWorker")
77
- elif self.speculative_config:
78
- worker_kwargs.update(
79
- worker_module_name="vllm.spec_decode.spec_decode_worker",
80
- worker_class_name="create_spec_worker")
81
- else:
82
- worker_kwargs.update(worker_module_name="vllm.worker.worker",
83
- worker_class_name="Worker")
85
+ (worker_module_name,
86
+ worker_class_name) = self._get_worker_module_and_class()
87
+ worker_kwargs.update(worker_module_name=worker_module_name,
88
+ worker_class_name=worker_class_name)
89
90
return worker_kwargs
91
vllm/executor/ray_gpu_executor.py
+11
-10
@@ -91,15 +91,8 @@ class RayGPUExecutor(DistributedGPUExecutor):
91
return ray_remote_kwargs
92
93
def _get_worker_wrapper_args(self) -> Dict[str, Any]:
94
- if self.speculative_config is not None:
95
- worker_module_name = "vllm.spec_decode.spec_decode_worker"
96
- worker_class_name = "create_spec_worker"
97
- elif self.scheduler_config.is_multi_step:
98
- worker_module_name = "vllm.worker.multi_step_worker"
99
- worker_class_name = "MultiStepWorker"
100
- else:
101
- worker_module_name = "vllm.worker.worker"
102
- worker_class_name = "Worker"
94
+ (worker_module_name,
95
+ worker_class_name) = self._get_worker_module_and_class()
96
97
return dict(
98
worker_module_name=worker_module_name,
@@ -107,6 +100,10 @@ class RayGPUExecutor(DistributedGPUExecutor):
100
trust_remote_code=self.model_config.trust_remote_code,
101
)
102
103
+ # child class could overwrite this to return actual env vars.
104
+ def _get_env_vars_to_be_updated(self):
105
+ return self._env_vars_for_all_workers
106
+
107
def _init_workers_ray(self, placement_group: "PlacementGroup",
108
**ray_remote_kwargs):
109
if (self.parallel_config.tensor_parallel_size == 1
@@ -231,8 +228,12 @@ class RayGPUExecutor(DistributedGPUExecutor):
228
"VLLM_TRACE_FUNCTION":
229
str(envs.VLLM_TRACE_FUNCTION),
230
}, ) for (node_id, _) in worker_node_and_gpu_ids]
231
+
232
+ self._env_vars_for_all_workers = (
233
+ all_args_to_update_environment_variables)
234
+
235
self._run_workers("update_environment_variables",
235
- all_args=all_args_to_update_environment_variables)
236
+ all_args=self._get_env_vars_to_be_updated())
237
238
if len(node_gpus) == 1:
239
# in single node case, we don't need to get the IP address.