@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d87f39e9a9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] Add init_cached_hf_modules to RayWorkerWrapper (#4286)

DefTruth committed Apr 24, 2024 at 00:28 UTC d87f39e9a9dd149f5dd7a58b4d98b21f713827b6
2 files changed +8 -1
vllm/executor/ray_gpu_executor.py
+2
@@ -100,6 +100,7 @@ class RayGPUExecutor(ExecutorBase):
100 )(RayWorkerWrapper).remote(
101 worker_module_name="vllm.worker.worker",
102 worker_class_name="Worker",
103 + trust_remote_code=self.model_config.trust_remote_code,
104 )
105
106 worker_ip = ray.get(worker.get_node_ip.remote())
@@ -110,6 +111,7 @@ class RayGPUExecutor(ExecutorBase):
111 self.driver_worker = RayWorkerWrapper(
112 worker_module_name="vllm.worker.worker",
113 worker_class_name="Worker",
114 + trust_remote_code=self.model_config.trust_remote_code,
115 )
116 else:
117 # Else, added to the list of workers.
vllm/worker/worker_base.py
+6 -1
@@ -103,10 +103,15 @@ class WorkerWrapperBase:
103
104 def __init__(self,
105 worker_module_name=None,
106 - worker_class_name=None) -> None:
106 + worker_class_name=None,
107 + trust_remote_code: bool = False) -> None:
108 self.worker_module_name = worker_module_name
109 self.worker_class_name = worker_class_name
110 self.worker = None
111 + if trust_remote_code:
112 + # note: lazy import to avoid importing torch before initializing
113 + from vllm.utils import init_cached_hf_modules
114 + init_cached_hf_modules()
115
116 @staticmethod
117 def update_environment_variables(envs: Dict[str, str]) -> None: