@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9118217f58
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[LoRA] Relax LoRA condition (#7146)

Jee Jee Li committed Aug 6, 2024 at 09:57 UTC 9118217f58c39040aa935b7c85250c7364ffa72d
4 files changed +8 -7
tests/lora/test_layers.py
+1 -1
@@ -420,7 +420,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
420 @torch.inference_mode()
421 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
422 @pytest.mark.parametrize("device", CUDA_DEVICES)
423 -@pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
423 +@pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 256512])
424 @pytest.mark.parametrize("stage", STAGES)
425 def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
426 stage) -> None:
tests/lora/test_punica_variation.py
+1 -1
@@ -25,7 +25,7 @@ HIDDEN_SIZES = [3424, 4096, 4097]
25 BATCHES = [1, 4, 16, 32]
26 NUM_LORA = [1, 4, 8, 16, 32, 64, 128]
27 DTYPES = [torch.float16, torch.bfloat16]
28 -MAX_RANKS = [1, 4, 8, 16, 32, 64, 128]
28 +MAX_RANKS = [1, 4, 8, 16, 32, 64, 128, 256]
29 SCALES = [0.5]
30 SEED = [0]
31 CUDA_DEVICES = [f"cuda:{0}"]
vllm/config.py
+3 -2
@@ -1311,8 +1311,9 @@ class LoRAConfig:
1311 long_lora_scaling_factors: Optional[Tuple[float]] = None
1312
1313 def __post_init__(self):
1314 - # TODO: Increase the range of rank
1315 - possible_max_ranks = (8, 16, 32, 64)
1314 + # Setting the maximum rank to 256 should be able to satisfy the vast
1315 + # majority of applications.
1316 + possible_max_ranks = (8, 16, 32, 64, 128, 256)
1317 possible_lora_extra_vocab_size = (0, 256, 512)
1318 if self.max_lora_rank not in possible_max_ranks:
1319 raise ValueError(
vllm/lora/layers.py
+3 -3
@@ -1073,10 +1073,10 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1073 lora_config: LoRAConfig,
1074 model_config: Optional[PretrainedConfig] = None,
1075 ) -> None:
1076 - # TODO: Verify if this condition can be relaxed
1077 - if 32000 < self.base_layer.vocab_size > 128512:
1076 + # TODO: Verify if this condition can be further relaxed
1077 + if 32000 < self.base_layer.vocab_size > 257024:
1078 raise ValueError("When using LoRA, vocab size must be "
1079 - "32000 >= vocab_size <= 128512")
1079 + "32000 >= vocab_size <= 257024")
1080 self.lora_a_stacked = torch.zeros(
1081 (
1082 max_loras,