@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b6f99a6ffe
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Core] Refactor executor classes for easier inheritance (#7673)

[Core] Refactor executor classes to make it easier to inherit GPUExecutor (#7673)

Kunshang Ji committed Aug 20, 2024 at 15:56 UTC b6f99a6ffe4c7471b32932a0d163fe0e7bbead5b
2 files changed +27 -21
vllm/executor/gpu_executor.py
+16 -11
@@ -62,6 +62,18 @@ class GPUExecutor(ExecutorBase):
62 observability_config=self.observability_config,
63 )
64
65 + def _get_worker_module_and_class(self) -> Tuple[str, str]:
66 + if self.scheduler_config.is_multi_step:
67 + worker_module_name = "vllm.worker.multi_step_worker"
68 + worker_class_name = "MultiStepWorker"
69 + elif self.speculative_config:
70 + worker_module_name = "vllm.spec_decode.spec_decode_worker"
71 + worker_class_name = "create_spec_worker"
72 + else:
73 + worker_module_name = "vllm.worker.worker"
74 + worker_class_name = "Worker"
75 + return (worker_module_name, worker_class_name)
76 +
77 def _get_create_worker_kwargs(
78 self,
79 local_rank: int = 0,
@@ -70,17 +82,10 @@ class GPUExecutor(ExecutorBase):
82 worker_kwargs = self._get_worker_kwargs(local_rank, rank,
83 distributed_init_method)
84
73 - if self.scheduler_config.is_multi_step:
74 - worker_kwargs.update(
75 - worker_module_name="vllm.worker.multi_step_worker",
76 - worker_class_name="MultiStepWorker")
77 - elif self.speculative_config:
78 - worker_kwargs.update(
79 - worker_module_name="vllm.spec_decode.spec_decode_worker",
80 - worker_class_name="create_spec_worker")
81 - else:
82 - worker_kwargs.update(worker_module_name="vllm.worker.worker",
83 - worker_class_name="Worker")
85 + (worker_module_name,
86 + worker_class_name) = self._get_worker_module_and_class()
87 + worker_kwargs.update(worker_module_name=worker_module_name,
88 + worker_class_name=worker_class_name)
89
90 return worker_kwargs
91
vllm/executor/ray_gpu_executor.py
+11 -10
@@ -91,15 +91,8 @@ class RayGPUExecutor(DistributedGPUExecutor):
91 return ray_remote_kwargs
92
93 def _get_worker_wrapper_args(self) -> Dict[str, Any]:
94 - if self.speculative_config is not None:
95 - worker_module_name = "vllm.spec_decode.spec_decode_worker"
96 - worker_class_name = "create_spec_worker"
97 - elif self.scheduler_config.is_multi_step:
98 - worker_module_name = "vllm.worker.multi_step_worker"
99 - worker_class_name = "MultiStepWorker"
100 - else:
101 - worker_module_name = "vllm.worker.worker"
102 - worker_class_name = "Worker"
94 + (worker_module_name,
95 + worker_class_name) = self._get_worker_module_and_class()
96
97 return dict(
98 worker_module_name=worker_module_name,
@@ -107,6 +100,10 @@ class RayGPUExecutor(DistributedGPUExecutor):
100 trust_remote_code=self.model_config.trust_remote_code,
101 )
102
103 + # child class could overwrite this to return actual env vars.
104 + def _get_env_vars_to_be_updated(self):
105 + return self._env_vars_for_all_workers
106 +
107 def _init_workers_ray(self, placement_group: "PlacementGroup",
108 **ray_remote_kwargs):
109 if (self.parallel_config.tensor_parallel_size == 1
@@ -231,8 +228,12 @@ class RayGPUExecutor(DistributedGPUExecutor):
228 "VLLM_TRACE_FUNCTION":
229 str(envs.VLLM_TRACE_FUNCTION),
230 }, ) for (node_id, _) in worker_node_and_gpu_ids]
231 +
232 + self._env_vars_for_all_workers = (
233 + all_args_to_update_environment_variables)
234 +
235 self._run_workers("update_environment_variables",
235 - all_args=all_args_to_update_environment_variables)
236 + all_args=self._get_env_vars_to_be_updated())
237
238 if len(node_gpus) == 1:
239 # in single node case, we don't need to get the IP address.