A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[Bugfix] Fix LoRA extra vocab size (#15047)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Mar 19, 2025 at 00:40 UTC
46c759c165a5a985ce62f019bf684e4a6109e41c
5 files changed
+1
-5
examples/offline_inference/audio_language.py
-1
@@ -93,7 +93,6 @@ def run_phi4mm(question: str, audio_count: int) -> ModelRequestData:
93
max_num_seqs=2,
94
enable_lora=True,
95
max_lora_rank=320,
96
- lora_extra_vocab_size=0,
96
limit_mm_per_prompt={"audio": audio_count},
97
)
98
examples/offline_inference/vision_language.py
-1
@@ -682,7 +682,6 @@ def run_phi4mm(questions: list[str], modality: str) -> ModelRequestData:
682
max_num_seqs=2,
683
enable_lora=True,
684
max_lora_rank=320,
685
- lora_extra_vocab_size=0,
685
)
686
687
return ModelRequestData(
examples/offline_inference/vision_language_multi_image.py
-1
@@ -342,7 +342,6 @@ def load_phi4mm(question: str, image_urls: list[str]) -> ModelRequestData:
342
limit_mm_per_prompt={"image": len(image_urls)},
343
enable_lora=True,
344
max_lora_rank=320,
345
- lora_extra_vocab_size=0,
345
)
346
347
placeholders = "".join(f"<|image_{i}|>"
tests/models/decoder_only/vision_language/test_phi4mm.py
-1
@@ -100,7 +100,6 @@ def run_test(
100
distributed_executor_backend=distributed_executor_backend,
101
enable_lora=True,
102
max_lora_rank=320,
103
- lora_extra_vocab_size=0,
103
gpu_memory_utilization=0.8, # set to 0.8 to avoid OOM in CI
104
enforce_eager=True,
105
) as vllm_model:
vllm/config.py
+1
-1
@@ -2324,7 +2324,7 @@ class LoRAConfig:
2324
# Setting the maximum rank to 512 should be able to satisfy the vast
2325
# majority of applications.
2326
possible_max_ranks = (8, 16, 32, 64, 128, 256, 320, 512)
2327
- possible_lora_extra_vocab_size = (0, 256, 512)
2327
+ possible_lora_extra_vocab_size = (256, 512)
2328
if self.max_lora_rank not in possible_max_ranks:
2329
raise ValueError(
2330
f"max_lora_rank ({self.max_lora_rank}) must be one of "