A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] Add init_cached_hf_modules to RayWorkerWrapper (#4286)
DefTruth committed
Apr 24, 2024 at 00:28 UTC
d87f39e9a9dd149f5dd7a58b4d98b21f713827b6
2 files changed
+8
-1
vllm/executor/ray_gpu_executor.py
+2
@@ -100,6 +100,7 @@ class RayGPUExecutor(ExecutorBase):
100
)(RayWorkerWrapper).remote(
101
worker_module_name="vllm.worker.worker",
102
worker_class_name="Worker",
103
+ trust_remote_code=self.model_config.trust_remote_code,
104
)
105
106
worker_ip = ray.get(worker.get_node_ip.remote())
@@ -110,6 +111,7 @@ class RayGPUExecutor(ExecutorBase):
111
self.driver_worker = RayWorkerWrapper(
112
worker_module_name="vllm.worker.worker",
113
worker_class_name="Worker",
114
+ trust_remote_code=self.model_config.trust_remote_code,
115
)
116
else:
117
# Else, added to the list of workers.
vllm/worker/worker_base.py
+6
-1
@@ -103,10 +103,15 @@ class WorkerWrapperBase:
103
104
def __init__(self,
105
worker_module_name=None,
106
- worker_class_name=None) -> None:
106
+ worker_class_name=None,
107
+ trust_remote_code: bool = False) -> None:
108
self.worker_module_name = worker_module_name
109
self.worker_class_name = worker_class_name
110
self.worker = None
111
+ if trust_remote_code:
112
+ # note: lazy import to avoid importing torch before initializing
113
+ from vllm.utils import init_cached_hf_modules
114
+ init_cached_hf_modules()
115
116
@staticmethod
117
def update_environment_variables(envs: Dict[str, str]) -> None: