A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[CI/Build] Fix LoRA OOM (#16624)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Apr 15, 2025 at 16:38 UTC
1575c1701a80befec8efe274b338cb26bc199275
1 file changed
+13
-4
tests/lora/test_minicpmv_tp.py
+13
-4
@@ -66,8 +66,12 @@ def test_minicpmv_lora(minicpmv_lora_files):
66
max_loras=2,
67
max_lora_rank=8,
68
enforce_eager=True,
69
+ max_model_len=2048,
70
+ limit_mm_per_prompt={
71
+ "image": 2,
72
+ "video": 0
73
+ },
74
trust_remote_code=True,
70
- enable_chunked_prefill=True,
75
)
76
output1 = do_sample(llm, minicpmv_lora_files, lora_id=1)
77
for i in range(len(EXPECTED_OUTPUT)):
@@ -91,9 +95,11 @@ def test_minicpmv_tp4_wo_fully_sharded_loras(minicpmv_lora_files):
95
max_loras=4,
96
max_lora_rank=64,
97
tensor_parallel_size=4,
98
+ limit_mm_per_prompt={
99
+ "image": 2,
100
+ "video": 0
101
+ },
102
trust_remote_code=True,
95
- enforce_eager=True,
96
- enable_chunked_prefill=True,
103
)
104
output_tp = do_sample(llm, minicpmv_lora_files, lora_id=1)
105
for i in range(len(EXPECTED_OUTPUT)):
@@ -115,8 +121,11 @@ def test_minicpmv_tp4_fully_sharded_loras(minicpmv_lora_files):
121
max_lora_rank=8,
122
tensor_parallel_size=4,
123
trust_remote_code=True,
124
+ limit_mm_per_prompt={
125
+ "image": 1,
126
+ "video": 0
127
+ },
128
fully_sharded_loras=True,
119
- enable_chunked_prefill=True,
129
)
130
output_tp = do_sample(llm, minicpmv_lora_files, lora_id=1)
131
for i in range(len(EXPECTED_OUTPUT)):