A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[Misc] V1 LoRA support CPU offload (#15843)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Apr 2, 2025 at 23:04 UTC
58e234a7542f6df2e769eaab4f1661a3a7494441
1 file changed
+3
-3
vllm/config.py
+3
-3
@@ -2434,9 +2434,9 @@ class LoRAConfig:
2434
f"max_loras ({self.max_loras})")
2435
2436
def verify_with_cache_config(self, cache_config: CacheConfig):
2437
- # TODO LoRA supports CPU offload.
2438
- if cache_config.cpu_offload_gb > 0:
2439
- raise ValueError("CPU offload is not supported with LoRA yet.")
2437
+ if cache_config.cpu_offload_gb > 0 and not envs.VLLM_USE_V1:
2438
+ raise ValueError(
2439
+ "V0 LoRA does not support CPU offload, please use V1.")
2440
2441
def verify_with_model_config(self, model_config: ModelConfig):
2442
if self.lora_dtype in (None, "auto"):