A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
fix ModuleNotFoundError (#599)
Co-authored-by: fangli <fangli@tencent.com>
Fang li committed
Jul 30, 2023 at 11:52 UTC
953f28cf9a8440f3b61bb99013c12638368581ec
1 file changed
+7
-3
vllm/engine/llm_engine.py
+7
-3
@@ -1,4 +1,5 @@
1
import time
2
+import copy
3
from functools import partial
4
from typing import Any, List, Optional, TYPE_CHECKING
5
@@ -145,12 +146,15 @@ class LLMEngine:
146
147
# Initialize torch distributed process group for the workers.
148
init_torch_dist_process_group(self.workers, backend="nccl")
149
+ model_config = copy.deepcopy(self.model_config)
150
+ parallel_config = copy.deepcopy(self.parallel_config)
151
+ scheduler_config = copy.deepcopy(self.scheduler_config)
152
self._run_workers("init_worker",
153
get_all_outputs=True,
154
worker_init_fn=lambda: Worker(
151
- self.model_config,
152
- self.parallel_config,
153
- self.scheduler_config,
155
+ model_config,
156
+ parallel_config,
157
+ scheduler_config,
158
None,
159
None,
160
))