A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Fix error when tp > 1 (#2644)
Co-authored-by: zhaoyang-star <zhao.yang16@zte.com.cn>
zhaoyang-star committed
Jan 29, 2024 at 14:47 UTC
b72af8f1eded6f5838be29eb6093ab0e0e0c240c
1 file changed
+2
-3
vllm/engine/llm_engine.py
+2
-3
@@ -236,7 +236,6 @@ class LLMEngine:
236
model_config = copy.deepcopy(self.model_config)
237
parallel_config = copy.deepcopy(self.parallel_config)
238
scheduler_config = copy.deepcopy(self.scheduler_config)
239
- cache_config = copy.deepcopy(self.cache_config)
239
240
for rank, (worker, (node_id,
241
_)) in enumerate(zip(self.workers,
@@ -252,7 +251,7 @@ class LLMEngine:
251
rank,
252
distributed_init_method,
253
lora_config=self.lora_config,
255
- cache_config=cache_config,
254
+ kv_cache_dtype=self.cache_config.cache_dtype,
255
))
256
257
driver_rank = 0
@@ -265,7 +264,7 @@ class LLMEngine:
264
driver_rank,
265
distributed_init_method,
266
lora_config=self.lora_config,
268
- cache_config=cache_config,
267
+ kv_cache_dtype=self.cache_config.cache_dtype,
268
is_driver_worker=True,
269
)
270