A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix] Add checks for LoRA and CPU offload (#11810)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Jan 8, 2025 at 13:08 UTC
f645eb69545672d394e9e9e0ce46c725504fd2a0
1 file changed
+6
vllm/config.py
+6
@@ -2051,6 +2051,11 @@ class LoRAConfig:
2051
f"max_cpu_loras ({self.max_cpu_loras}) must be >= "
2052
f"max_loras ({self.max_loras})")
2053
2054
+ def verify_with_cache_config(self, cache_config: CacheConfig):
2055
+ # TODO LoRA supports CPU offload.
2056
+ if cache_config.cpu_offload_gb > 0:
2057
+ raise ValueError("CPU offload is not supported with LoRA yet.")
2058
+
2059
def verify_with_model_config(self, model_config: ModelConfig):
2060
if self.lora_dtype in (None, "auto"):
2061
self.lora_dtype = model_config.dtype
@@ -3138,6 +3143,7 @@ class VllmConfig:
3143
self.cache_config.verify_with_parallel_config(self.parallel_config)
3144
3145
if self.lora_config:
3146
+ self.lora_config.verify_with_cache_config(self.cache_config)
3147
self.lora_config.verify_with_model_config(self.model_config)
3148
self.lora_config.verify_with_scheduler_config(
3149
self.scheduler_config)