@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b72af8f1ed
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Fix error when tp > 1 (#2644)

Co-authored-by: zhaoyang-star <zhao.yang16@zte.com.cn>

zhaoyang-star committed Jan 29, 2024 at 14:47 UTC b72af8f1eded6f5838be29eb6093ab0e0e0c240c
1 file changed +2 -3
vllm/engine/llm_engine.py
+2 -3
@@ -236,7 +236,6 @@ class LLMEngine:
236 model_config = copy.deepcopy(self.model_config)
237 parallel_config = copy.deepcopy(self.parallel_config)
238 scheduler_config = copy.deepcopy(self.scheduler_config)
239 - cache_config = copy.deepcopy(self.cache_config)
239
240 for rank, (worker, (node_id,
241 _)) in enumerate(zip(self.workers,
@@ -252,7 +251,7 @@ class LLMEngine:
251 rank,
252 distributed_init_method,
253 lora_config=self.lora_config,
255 - cache_config=cache_config,
254 + kv_cache_dtype=self.cache_config.cache_dtype,
255 ))
256
257 driver_rank = 0
@@ -265,7 +264,7 @@ class LLMEngine:
264 driver_rank,
265 distributed_init_method,
266 lora_config=self.lora_config,
268 - cache_config=cache_config,
267 + kv_cache_dtype=self.cache_config.cache_dtype,
268 is_driver_worker=True,
269 )
270