A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Core] Relax the LoRA max rank (#26461)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Oct 9, 2025 at 14:47 UTC
1b2c440cd646dac290b535b86be89d22fbdbeab9
2 files changed
+5
-5
vllm/config/lora.py
+1
-1
@@ -103,7 +103,7 @@ class LoRAConfig:
103
104
# Setting the maximum rank to 512 should be able to satisfy the vast
105
# majority of applications.
106
- possible_max_ranks = (8, 16, 32, 64, 128, 256, 320, 512)
106
+ possible_max_ranks = (1, 8, 16, 32, 64, 128, 256, 320, 512)
107
possible_lora_extra_vocab_size = (256, 512)
108
if self.max_lora_rank not in possible_max_ranks:
109
raise ValueError(
vllm/v1/worker/lora_model_runner_mixin.py
+4
-4
@@ -28,8 +28,6 @@ logger = init_logger(__name__)
28
29
# Defined as a mixin for GPUModelRunner
30
class LoRAModelRunnerMixin:
31
- LORA_WARMUP_RANK = 8
32
-
31
def load_lora_model(
32
self, model: nn.Module, vllm_config: VllmConfig, device: torch.device
33
) -> nn.Module:
@@ -96,7 +94,9 @@ class LoRAModelRunnerMixin:
94
assert self.lora_manager is not None, "LoRA is not enabled"
95
96
num_loras = lora_config.max_loras
99
-
97
+ lora_warmup_rank = (
98
+ lora_config.max_lora_rank if lora_config.max_lora_rank < 8 else 8
99
+ )
100
# Make dummy lora requests
101
lora_requests: set[LoRARequest] = {
102
LoRARequest(
@@ -111,7 +111,7 @@ class LoRAModelRunnerMixin:
111
# Add the dummy LoRAs here so _set_active_loras doesn't try to
112
# load from disk.
113
for lr in lora_requests:
114
- self.lora_manager.add_dummy_lora(lr, rank=self.LORA_WARMUP_RANK)
114
+ self.lora_manager.add_dummy_lora(lr, rank=lora_warmup_rank)
115
116
yield
117