A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[LoRA] Relax LoRA condition (#7146)
Jee Jee Li committed
Aug 6, 2024 at 09:57 UTC
9118217f58c39040aa935b7c85250c7364ffa72d
4 files changed
+8
-7
tests/lora/test_layers.py
+1
-1
@@ -420,7 +420,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
420
@torch.inference_mode()
421
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
422
@pytest.mark.parametrize("device", CUDA_DEVICES)
423
-@pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
423
+@pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 256512])
424
@pytest.mark.parametrize("stage", STAGES)
425
def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
426
stage) -> None:
tests/lora/test_punica_variation.py
+1
-1
@@ -25,7 +25,7 @@ HIDDEN_SIZES = [3424, 4096, 4097]
25
BATCHES = [1, 4, 16, 32]
26
NUM_LORA = [1, 4, 8, 16, 32, 64, 128]
27
DTYPES = [torch.float16, torch.bfloat16]
28
-MAX_RANKS = [1, 4, 8, 16, 32, 64, 128]
28
+MAX_RANKS = [1, 4, 8, 16, 32, 64, 128, 256]
29
SCALES = [0.5]
30
SEED = [0]
31
CUDA_DEVICES = [f"cuda:{0}"]
vllm/config.py
+3
-2
@@ -1311,8 +1311,9 @@ class LoRAConfig:
1311
long_lora_scaling_factors: Optional[Tuple[float]] = None
1312
1313
def __post_init__(self):
1314
- # TODO: Increase the range of rank
1315
- possible_max_ranks = (8, 16, 32, 64)
1314
+ # Setting the maximum rank to 256 should be able to satisfy the vast
1315
+ # majority of applications.
1316
+ possible_max_ranks = (8, 16, 32, 64, 128, 256)
1317
possible_lora_extra_vocab_size = (0, 256, 512)
1318
if self.max_lora_rank not in possible_max_ranks:
1319
raise ValueError(
vllm/lora/layers.py
+3
-3
@@ -1073,10 +1073,10 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1073
lora_config: LoRAConfig,
1074
model_config: Optional[PretrainedConfig] = None,
1075
) -> None:
1076
- # TODO: Verify if this condition can be relaxed
1077
- if 32000 < self.base_layer.vocab_size > 128512:
1076
+ # TODO: Verify if this condition can be further relaxed
1077
+ if 32000 < self.base_layer.vocab_size > 257024:
1078
raise ValueError("When using LoRA, vocab size must be "
1079
- "32000 >= vocab_size <= 128512")
1079
+ "32000 >= vocab_size <= 257024")
1080
self.lora_a_stacked = torch.zeros(
1081
(
1082
max_loras,