@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 953f28cf9a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

fix ModuleNotFoundError (#599)

Co-authored-by: fangli <fangli@tencent.com>

Fang li committed Jul 30, 2023 at 11:52 UTC 953f28cf9a8440f3b61bb99013c12638368581ec
1 file changed +7 -3
vllm/engine/llm_engine.py
+7 -3
@@ -1,4 +1,5 @@
1 import time
2 +import copy
3 from functools import partial
4 from typing import Any, List, Optional, TYPE_CHECKING
5
@@ -145,12 +146,15 @@ class LLMEngine:
146
147 # Initialize torch distributed process group for the workers.
148 init_torch_dist_process_group(self.workers, backend="nccl")
149 + model_config = copy.deepcopy(self.model_config)
150 + parallel_config = copy.deepcopy(self.parallel_config)
151 + scheduler_config = copy.deepcopy(self.scheduler_config)
152 self._run_workers("init_worker",
153 get_all_outputs=True,
154 worker_init_fn=lambda: Worker(
151 - self.model_config,
152 - self.parallel_config,
153 - self.scheduler_config,
155 + model_config,
156 + parallel_config,
157 + scheduler_config,
158 None,
159 None,
160 ))