@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f645eb6954
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix] Add checks for LoRA and CPU offload (#11810)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Jan 8, 2025 at 13:08 UTC f645eb69545672d394e9e9e0ce46c725504fd2a0
1 file changed +6
vllm/config.py
+6
@@ -2051,6 +2051,11 @@ class LoRAConfig:
2051 f"max_cpu_loras ({self.max_cpu_loras}) must be >= "
2052 f"max_loras ({self.max_loras})")
2053
2054 + def verify_with_cache_config(self, cache_config: CacheConfig):
2055 + # TODO LoRA supports CPU offload.
2056 + if cache_config.cpu_offload_gb > 0:
2057 + raise ValueError("CPU offload is not supported with LoRA yet.")
2058 +
2059 def verify_with_model_config(self, model_config: ModelConfig):
2060 if self.lora_dtype in (None, "auto"):
2061 self.lora_dtype = model_config.dtype
@@ -3138,6 +3143,7 @@ class VllmConfig:
3143 self.cache_config.verify_with_parallel_config(self.parallel_config)
3144
3145 if self.lora_config:
3146 + self.lora_config.verify_with_cache_config(self.cache_config)
3147 self.lora_config.verify_with_model_config(self.model_config)
3148 self.lora_config.verify_with_scheduler_config(
3149 self.scheduler_config)