@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4042d192f5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

fix "tansformers_module" ModuleNotFoundError when load model with `trust_remote_code=True` (#871)

Jingru committed Sep 9, 2023 at 08:21 UTC 4042d192f5fbad6c95563f2880cd30dac8caf8f2
2 files changed +6 -2
vllm/engine/llm_engine.py
+1 -1
@@ -153,7 +153,7 @@ class LLMEngine:
153 placement_group=placement_group,
154 placement_group_capture_child_tasks=True),
155 **ray_remote_kwargs,
156 - )(RayWorker).remote()
156 + )(RayWorker).remote(self.model_config.trust_remote_code)
157 self.workers.append(worker)
158
159 # Initialize torch distributed process group for the workers.
vllm/engine/ray_utils.py
+5 -1
@@ -11,7 +11,11 @@ try:
11 """Ray wrapper for vllm.worker.Worker, allowing Worker to be
12 lazliy initialized after Ray sets CUDA_VISIBLE_DEVICES."""
13
14 - def __init__(self) -> None:
14 + def __init__(self, init_cached_hf_modules=False) -> None:
15 + if init_cached_hf_modules:
16 + # pylint: disable=import-outside-toplevel
17 + from transformers.dynamic_module_utils import init_hf_modules
18 + init_hf_modules()
19 self.worker = None
20
21 def init_worker(self, worker_init_fn):