@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d5c5154fcf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Misc] LoRA + Chunked Prefill (#9057)

Aurick Qiao committed Dec 10, 2024 at 21:09 UTC d5c5154fcf4c5d65551c98e458cbb027e5f4b672
12 files changed +49 -20
tests/lora/test_chatglm3_tp.py
+6 -3
@@ -53,7 +53,8 @@ def test_chatglm3_lora(chatglm3_lora_files):
53 max_loras=4,
54 max_lora_rank=64,
55 tensor_parallel_size=1,
56 - trust_remote_code=True)
56 + trust_remote_code=True,
57 + enable_chunked_prefill=True)
58
59 output1 = do_sample(llm, chatglm3_lora_files, lora_id=1)
60 for i in range(len(EXPECTED_LORA_OUTPUT)):
@@ -73,7 +74,8 @@ def test_chatglm3_lora_tp4(chatglm3_lora_files):
74 max_lora_rank=64,
75 tensor_parallel_size=4,
76 trust_remote_code=True,
76 - fully_sharded_loras=False)
77 + fully_sharded_loras=False,
78 + enable_chunked_prefill=True)
79
80 output1 = do_sample(llm, chatglm3_lora_files, lora_id=1)
81 for i in range(len(EXPECTED_LORA_OUTPUT)):
@@ -93,7 +95,8 @@ def test_chatglm3_lora_tp4_fully_sharded_loras(chatglm3_lora_files):
95 max_lora_rank=64,
96 tensor_parallel_size=4,
97 trust_remote_code=True,
96 - fully_sharded_loras=True)
98 + fully_sharded_loras=True,
99 + enable_chunked_prefill=True)
100 output1 = do_sample(llm, chatglm3_lora_files, lora_id=1)
101 for i in range(len(EXPECTED_LORA_OUTPUT)):
102 assert output1[i] == EXPECTED_LORA_OUTPUT[i]
tests/lora/test_gemma.py
+2 -1
@@ -37,7 +37,8 @@ def test_gemma_lora(gemma_lora_files):
37 llm = vllm.LLM(MODEL_PATH,
38 max_model_len=1024,
39 enable_lora=True,
40 - max_loras=4)
40 + max_loras=4,
41 + enable_chunked_prefill=True)
42
43 expected_lora_output = [
44 "more important than knowledge.\nAuthor: Albert Einstein\n",
tests/lora/test_llama_tp.py
+5 -1
@@ -78,7 +78,8 @@ def test_llama_lora(sql_lora_files):
78 enable_lora=True,
79 max_num_seqs=16,
80 max_loras=4,
81 - tensor_parallel_size=1)
81 + tensor_parallel_size=1,
82 + enable_chunked_prefill=True)
83 generate_and_test(llm, sql_lora_files)
84
85
@@ -120,6 +121,7 @@ def test_llama_lora_tp4(sql_lora_files):
121 max_num_seqs=16,
122 max_loras=4,
123 tensor_parallel_size=4,
124 + enable_chunked_prefill=True,
125 )
126 generate_and_test(llm, sql_lora_files)
127
@@ -135,6 +137,7 @@ def test_llama_lora_tp4_fully_sharded_loras(sql_lora_files):
137 max_loras=4,
138 tensor_parallel_size=4,
139 fully_sharded_loras=True,
140 + enable_chunked_prefill=True,
141 )
142 generate_and_test(llm, sql_lora_files)
143
@@ -151,5 +154,6 @@ def test_llama_lora_tp4_fully_sharded_enable_bias(sql_lora_files):
154 tensor_parallel_size=4,
155 fully_sharded_loras=True,
156 enable_lora_bias=True,
157 + enable_chunked_prefill=True,
158 )
159 generate_and_test(llm, sql_lora_files)
tests/lora/test_long_context.py
+2 -1
@@ -124,7 +124,8 @@ def lora_llm(long_context_infos):
124 tensor_parallel_size=4,
125 # FIXME enable async output processor
126 disable_async_output_proc=True,
127 - distributed_executor_backend="mp")
127 + distributed_executor_backend="mp",
128 + enable_chunked_prefill=True)
129 yield llm
130 del llm
131
tests/lora/test_minicpmv.py
+2 -1
@@ -67,7 +67,8 @@ def test_minicpmv_lora(minicpmv_lora_files):
67 max_loras=4,
68 max_lora_rank=64,
69 trust_remote_code=True,
70 - gpu_memory_utilization=0.97 # This model is pretty big for CI gpus
70 + gpu_memory_utilization=0.97, # This model is pretty big for CI gpus
71 + enable_chunked_prefill=True,
72 )
73 output1 = do_sample(llm, minicpmv_lora_files, lora_id=1)
74 for i in range(len(EXPECTED_OUTPUT)):
tests/lora/test_minicpmv_tp.py
+2
@@ -69,6 +69,7 @@ def test_minicpmv_tp2(minicpmv_lora_files, fully_sharded):
69 tensor_parallel_size=2,
70 trust_remote_code=True,
71 fully_sharded_loras=fully_sharded,
72 + enable_chunked_prefill=True,
73 )
74
75 output_tp = do_sample(llm, minicpmv_lora_files, lora_id=1)
@@ -89,6 +90,7 @@ def test_minicpmv_tp4(minicpmv_lora_files, fully_sharded):
90 tensor_parallel_size=4,
91 trust_remote_code=True,
92 fully_sharded_loras=fully_sharded,
93 + enable_chunked_prefill=True,
94 )
95 output_tp = do_sample(llm, minicpmv_lora_files, lora_id=1)
96 for i in range(len(EXPECTED_OUTPUT)):
tests/lora/test_mixtral.py
+1
@@ -47,6 +47,7 @@ def test_mixtral_lora(mixtral_lora_files, tp_size):
47 max_loras=4,
48 distributed_executor_backend="ray",
49 tensor_parallel_size=tp_size,
50 + enable_chunked_prefill=True,
51 )
52
53 expected_lora_output = [
tests/lora/test_phi.py
+2 -1
@@ -53,7 +53,8 @@ def test_phi2_lora(phi2_lora_files):
53 max_model_len=1024,
54 enable_lora=True,
55 max_loras=2,
56 - enforce_eager=True)
56 + enforce_eager=True,
57 + enable_chunked_prefill=True)
58
59 expected_lora_output = [
60 "SELECT catalog_publisher, COUNT(*) as num_catalogs FROM catalogs GROUP BY catalog_publisher ORDER BY num_catalogs DESC LIMIT 1;", # noqa: E501
tests/lora/test_quant_model.py
+6 -3
@@ -84,7 +84,8 @@ def test_quant_model_lora(tinyllama_lora_files, num_gpus_available, model,
84 tensor_parallel_size=tp_size,
85 gpu_memory_utilization=0.2, #avoid OOM
86 quantization=model.quantization,
87 - trust_remote_code=True)
87 + trust_remote_code=True,
88 + enable_chunked_prefill=True)
89
90 if model.quantization is None:
91 expected_no_lora_output = [
@@ -176,7 +177,8 @@ def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available,
177 tensor_parallel_size=1,
178 gpu_memory_utilization=0.2, #avoid OOM
179 quantization=model.quantization,
179 - trust_remote_code=True)
180 + trust_remote_code=True,
181 + enable_chunked_prefill=True)
182 output_tp1 = do_sample(llm_tp1, tinyllama_lora_files, lora_id=1)
183
184 del llm_tp1
@@ -189,7 +191,8 @@ def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available,
191 max_loras=4,
192 tensor_parallel_size=2,
193 gpu_memory_utilization=0.2, #avoid OOM
192 - quantization=model.quantization)
194 + quantization=model.quantization,
195 + enable_chunked_prefill=True)
196 output_tp2 = do_sample(llm_tp2, tinyllama_lora_files, lora_id=1)
197
198 del llm_tp2
vllm/config.py
+2 -1
@@ -1698,7 +1698,8 @@ class LoRAConfig:
1698 # Reminder: Please update docs/source/usage/compatibility_matrix.rst
1699 # If the feature combo become valid
1700 if scheduler_config.chunked_prefill_enabled:
1701 - raise ValueError("LoRA is not supported with chunked prefill yet.")
1701 + logger.warning("LoRA with chunked prefill is still experimental "
1702 + "and may be unstable.")
1703
1704
1705 @dataclass
vllm/core/scheduler.py
+12 -3
@@ -166,9 +166,18 @@ class SchedulerOutputs:
166 and not self.blocks_to_swap_out and not self.blocks_to_copy)
167
168 def _sort_by_lora_ids(self):
169 - self.scheduled_seq_groups = sorted(
170 - self.scheduled_seq_groups,
171 - key=lambda g: (g.seq_group.lora_int_id, g.seq_group.request_id))
169 + assert 0 <= self.num_prefill_groups <= len(self.scheduled_seq_groups)
170 +
171 + def key_fn(group: ScheduledSequenceGroup):
172 + key = (group.seq_group.lora_int_id, group.seq_group.request_id)
173 + if 0 < self.num_prefill_groups < len(self.scheduled_seq_groups):
174 + # Sort sequence groups so that all prefills come before all
175 + # decodes as required by chunked prefill.
176 + return (not group.seq_group.is_prefill(), *key)
177 + return key
178 +
179 + self.scheduled_seq_groups = sorted(self.scheduled_seq_groups,
180 + key=key_fn)
181
182 @property
183 def lora_requests(self) -> Set[LoRARequest]:
vllm/worker/model_runner.py
+7 -5
@@ -622,11 +622,13 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
622 inter_data.lora_requests.add(seq_group_metadata.lora_request)
623 query_len = inter_data.query_lens[seq_idx]
624 inter_data.lora_index_mapping.append([lora_id] * query_len)
625 - inter_data.lora_prompt_mapping.append(
626 - [lora_id] *
627 - (query_len if seq_group_metadata.sampling_params
628 - and seq_group_metadata.sampling_params.prompt_logprobs is not None
629 - else 1))
625 + sampling_params = seq_group_metadata.sampling_params
626 + if sampling_params and sampling_params.prompt_logprobs is not None:
627 + inter_data.lora_prompt_mapping.append([lora_id] * query_len)
628 + elif not self.chunked_prefill_enabled or seq_group_metadata.do_sample:
629 + inter_data.lora_prompt_mapping.append([lora_id])
630 + else:
631 + inter_data.lora_prompt_mapping.append([])
632
633 def _compute_prompt_adapter_input(
634 self, inter_data: InterDataForSeqGroup,