@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cb3e73e4c8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[BugFix] fix wrong output when using lora and num_scheduler_steps=8 (#11161)

FIX issue https://github.com/vllm-project/vllm/issues/9688 https://github.com/vllm-project/vllm/issues/11086 #12487 --------- Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: weilong.yu <weilong.yu@shopee.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>

fade_away committed Feb 1, 2025 at 12:52 UTC cb3e73e4c8142b5ce8ac34efc2fa04d90f142dc5
2 files changed +4 -3
vllm/worker/model_runner.py
+4
@@ -1346,6 +1346,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1346
1347 self.execute_model(model_input, kv_caches, intermediate_tensors)
1348 torch.cuda.synchronize()
1349 + if self.lora_config:
1350 + # Remove dummy loras.
1351 + assert self.lora_manager is not None
1352 + self.remove_all_loras()
1353 return
1354
1355 def remove_all_loras(self):
vllm/worker/worker.py
-3
@@ -264,10 +264,7 @@ class Worker(LocalOrDistributedWorkerBase):
264 f"{(available_kv_cache_memory / GiB_bytes):.2f}GiB.")
265
266 logger.info(msg)
267 -
267 # Final cleanup
269 - if self.model_runner.lora_manager:
270 - self.model_runner.remove_all_loras()
268 gc.collect()
269
270 return num_gpu_blocks, num_cpu_blocks