A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
fix "tansformers_module" ModuleNotFoundError when load model with `trust_remote_code=True` (#871)
Jingru committed
Sep 9, 2023 at 08:21 UTC
4042d192f5fbad6c95563f2880cd30dac8caf8f2
2 files changed
+6
-2
vllm/engine/llm_engine.py
+1
-1
@@ -153,7 +153,7 @@ class LLMEngine:
153
placement_group=placement_group,
154
placement_group_capture_child_tasks=True),
155
**ray_remote_kwargs,
156
- )(RayWorker).remote()
156
+ )(RayWorker).remote(self.model_config.trust_remote_code)
157
self.workers.append(worker)
158
159
# Initialize torch distributed process group for the workers.
vllm/engine/ray_utils.py
+5
-1
@@ -11,7 +11,11 @@ try:
11
"""Ray wrapper for vllm.worker.Worker, allowing Worker to be
12
lazliy initialized after Ray sets CUDA_VISIBLE_DEVICES."""
13
14
- def __init__(self) -> None:
14
+ def __init__(self, init_cached_hf_modules=False) -> None:
15
+ if init_cached_hf_modules:
16
+ # pylint: disable=import-outside-toplevel
17
+ from transformers.dynamic_module_utils import init_hf_modules
18
+ init_hf_modules()
19
self.worker = None
20
21
def init_worker(self, worker_init_fn):