A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc] LoRA + Chunked Prefill (#9057)
Aurick Qiao committed
Dec 10, 2024 at 21:09 UTC
d5c5154fcf4c5d65551c98e458cbb027e5f4b672
12 files changed
+49
-20
tests/lora/test_chatglm3_tp.py
+6
-3
@@ -53,7 +53,8 @@ def test_chatglm3_lora(chatglm3_lora_files):
53
max_loras=4,
54
max_lora_rank=64,
55
tensor_parallel_size=1,
56
- trust_remote_code=True)
56
+ trust_remote_code=True,
57
+ enable_chunked_prefill=True)
58
59
output1 = do_sample(llm, chatglm3_lora_files, lora_id=1)
60
for i in range(len(EXPECTED_LORA_OUTPUT)):
@@ -73,7 +74,8 @@ def test_chatglm3_lora_tp4(chatglm3_lora_files):
74
max_lora_rank=64,
75
tensor_parallel_size=4,
76
trust_remote_code=True,
76
- fully_sharded_loras=False)
77
+ fully_sharded_loras=False,
78
+ enable_chunked_prefill=True)
79
80
output1 = do_sample(llm, chatglm3_lora_files, lora_id=1)
81
for i in range(len(EXPECTED_LORA_OUTPUT)):
@@ -93,7 +95,8 @@ def test_chatglm3_lora_tp4_fully_sharded_loras(chatglm3_lora_files):
95
max_lora_rank=64,
96
tensor_parallel_size=4,
97
trust_remote_code=True,
96
- fully_sharded_loras=True)
98
+ fully_sharded_loras=True,
99
+ enable_chunked_prefill=True)
100
output1 = do_sample(llm, chatglm3_lora_files, lora_id=1)
101
for i in range(len(EXPECTED_LORA_OUTPUT)):
102
assert output1[i] == EXPECTED_LORA_OUTPUT[i]
tests/lora/test_gemma.py
+2
-1
@@ -37,7 +37,8 @@ def test_gemma_lora(gemma_lora_files):
37
llm = vllm.LLM(MODEL_PATH,
38
max_model_len=1024,
39
enable_lora=True,
40
- max_loras=4)
40
+ max_loras=4,
41
+ enable_chunked_prefill=True)
42
43
expected_lora_output = [
44
"more important than knowledge.\nAuthor: Albert Einstein\n",
tests/lora/test_llama_tp.py
+5
-1
@@ -78,7 +78,8 @@ def test_llama_lora(sql_lora_files):
78
enable_lora=True,
79
max_num_seqs=16,
80
max_loras=4,
81
- tensor_parallel_size=1)
81
+ tensor_parallel_size=1,
82
+ enable_chunked_prefill=True)
83
generate_and_test(llm, sql_lora_files)
84
85
@@ -120,6 +121,7 @@ def test_llama_lora_tp4(sql_lora_files):
121
max_num_seqs=16,
122
max_loras=4,
123
tensor_parallel_size=4,
124
+ enable_chunked_prefill=True,
125
)
126
generate_and_test(llm, sql_lora_files)
127
@@ -135,6 +137,7 @@ def test_llama_lora_tp4_fully_sharded_loras(sql_lora_files):
137
max_loras=4,
138
tensor_parallel_size=4,
139
fully_sharded_loras=True,
140
+ enable_chunked_prefill=True,
141
)
142
generate_and_test(llm, sql_lora_files)
143
@@ -151,5 +154,6 @@ def test_llama_lora_tp4_fully_sharded_enable_bias(sql_lora_files):
154
tensor_parallel_size=4,
155
fully_sharded_loras=True,
156
enable_lora_bias=True,
157
+ enable_chunked_prefill=True,
158
)
159
generate_and_test(llm, sql_lora_files)
tests/lora/test_long_context.py
+2
-1
@@ -124,7 +124,8 @@ def lora_llm(long_context_infos):
124
tensor_parallel_size=4,
125
# FIXME enable async output processor
126
disable_async_output_proc=True,
127
- distributed_executor_backend="mp")
127
+ distributed_executor_backend="mp",
128
+ enable_chunked_prefill=True)
129
yield llm
130
del llm
131
tests/lora/test_minicpmv.py
+2
-1
@@ -67,7 +67,8 @@ def test_minicpmv_lora(minicpmv_lora_files):
67
max_loras=4,
68
max_lora_rank=64,
69
trust_remote_code=True,
70
- gpu_memory_utilization=0.97 # This model is pretty big for CI gpus
70
+ gpu_memory_utilization=0.97, # This model is pretty big for CI gpus
71
+ enable_chunked_prefill=True,
72
)
73
output1 = do_sample(llm, minicpmv_lora_files, lora_id=1)
74
for i in range(len(EXPECTED_OUTPUT)):
tests/lora/test_minicpmv_tp.py
+2
@@ -69,6 +69,7 @@ def test_minicpmv_tp2(minicpmv_lora_files, fully_sharded):
69
tensor_parallel_size=2,
70
trust_remote_code=True,
71
fully_sharded_loras=fully_sharded,
72
+ enable_chunked_prefill=True,
73
)
74
75
output_tp = do_sample(llm, minicpmv_lora_files, lora_id=1)
@@ -89,6 +90,7 @@ def test_minicpmv_tp4(minicpmv_lora_files, fully_sharded):
90
tensor_parallel_size=4,
91
trust_remote_code=True,
92
fully_sharded_loras=fully_sharded,
93
+ enable_chunked_prefill=True,
94
)
95
output_tp = do_sample(llm, minicpmv_lora_files, lora_id=1)
96
for i in range(len(EXPECTED_OUTPUT)):
tests/lora/test_mixtral.py
+1
@@ -47,6 +47,7 @@ def test_mixtral_lora(mixtral_lora_files, tp_size):
47
max_loras=4,
48
distributed_executor_backend="ray",
49
tensor_parallel_size=tp_size,
50
+ enable_chunked_prefill=True,
51
)
52
53
expected_lora_output = [
tests/lora/test_phi.py
+2
-1
@@ -53,7 +53,8 @@ def test_phi2_lora(phi2_lora_files):
53
max_model_len=1024,
54
enable_lora=True,
55
max_loras=2,
56
- enforce_eager=True)
56
+ enforce_eager=True,
57
+ enable_chunked_prefill=True)
58
59
expected_lora_output = [
60
"SELECT catalog_publisher, COUNT(*) as num_catalogs FROM catalogs GROUP BY catalog_publisher ORDER BY num_catalogs DESC LIMIT 1;", # noqa: E501
tests/lora/test_quant_model.py
+6
-3
@@ -84,7 +84,8 @@ def test_quant_model_lora(tinyllama_lora_files, num_gpus_available, model,
84
tensor_parallel_size=tp_size,
85
gpu_memory_utilization=0.2, #avoid OOM
86
quantization=model.quantization,
87
- trust_remote_code=True)
87
+ trust_remote_code=True,
88
+ enable_chunked_prefill=True)
89
90
if model.quantization is None:
91
expected_no_lora_output = [
@@ -176,7 +177,8 @@ def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available,
177
tensor_parallel_size=1,
178
gpu_memory_utilization=0.2, #avoid OOM
179
quantization=model.quantization,
179
- trust_remote_code=True)
180
+ trust_remote_code=True,
181
+ enable_chunked_prefill=True)
182
output_tp1 = do_sample(llm_tp1, tinyllama_lora_files, lora_id=1)
183
184
del llm_tp1
@@ -189,7 +191,8 @@ def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available,
191
max_loras=4,
192
tensor_parallel_size=2,
193
gpu_memory_utilization=0.2, #avoid OOM
192
- quantization=model.quantization)
194
+ quantization=model.quantization,
195
+ enable_chunked_prefill=True)
196
output_tp2 = do_sample(llm_tp2, tinyllama_lora_files, lora_id=1)
197
198
del llm_tp2
vllm/config.py
+2
-1
@@ -1698,7 +1698,8 @@ class LoRAConfig:
1698
# Reminder: Please update docs/source/usage/compatibility_matrix.rst
1699
# If the feature combo become valid
1700
if scheduler_config.chunked_prefill_enabled:
1701
- raise ValueError("LoRA is not supported with chunked prefill yet.")
1701
+ logger.warning("LoRA with chunked prefill is still experimental "
1702
+ "and may be unstable.")
1703
1704
1705
@dataclass
vllm/core/scheduler.py
+12
-3
@@ -166,9 +166,18 @@ class SchedulerOutputs:
166
and not self.blocks_to_swap_out and not self.blocks_to_copy)
167
168
def _sort_by_lora_ids(self):
169
- self.scheduled_seq_groups = sorted(
170
- self.scheduled_seq_groups,
171
- key=lambda g: (g.seq_group.lora_int_id, g.seq_group.request_id))
169
+ assert 0 <= self.num_prefill_groups <= len(self.scheduled_seq_groups)
170
+
171
+ def key_fn(group: ScheduledSequenceGroup):
172
+ key = (group.seq_group.lora_int_id, group.seq_group.request_id)
173
+ if 0 < self.num_prefill_groups < len(self.scheduled_seq_groups):
174
+ # Sort sequence groups so that all prefills come before all
175
+ # decodes as required by chunked prefill.
176
+ return (not group.seq_group.is_prefill(), *key)
177
+ return key
178
+
179
+ self.scheduled_seq_groups = sorted(self.scheduled_seq_groups,
180
+ key=key_fn)
181
182
@property
183
def lora_requests(self) -> Set[LoRARequest]:
vllm/worker/model_runner.py
+7
-5
@@ -622,11 +622,13 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
622
inter_data.lora_requests.add(seq_group_metadata.lora_request)
623
query_len = inter_data.query_lens[seq_idx]
624
inter_data.lora_index_mapping.append([lora_id] * query_len)
625
- inter_data.lora_prompt_mapping.append(
626
- [lora_id] *
627
- (query_len if seq_group_metadata.sampling_params
628
- and seq_group_metadata.sampling_params.prompt_logprobs is not None
629
- else 1))
625
+ sampling_params = seq_group_metadata.sampling_params
626
+ if sampling_params and sampling_params.prompt_logprobs is not None:
627
+ inter_data.lora_prompt_mapping.append([lora_id] * query_len)
628
+ elif not self.chunked_prefill_enabled or seq_group_metadata.do_sample:
629
+ inter_data.lora_prompt_mapping.append([lora_id])
630
+ else:
631
+ inter_data.lora_prompt_mapping.append([])
632
633
def _compute_prompt_adapter_input(
634
self, inter_data: InterDataForSeqGroup,