@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4d6ada947c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[CORE] Adding support for insertion of soft-tuned prompts (#4645)

Co-authored-by: Swapnil Parekh <swapnilp@ibm.com> Co-authored-by: Joe G <joseph.granados@h2o.ai> Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>

Swapnil Parekh committed Jul 9, 2024 at 16:26 UTC 4d6ada947c7e6379b6857bc9a9a1203679d32039
48 files changed +1951 -518
format.sh
+1
@@ -111,6 +111,7 @@ mypy vllm/spec_decode --config-file pyproject.toml
111 mypy vllm/model_executor --config-file pyproject.toml
112 mypy vllm/lora --config-file pyproject.toml
113 mypy vllm/logging --config-file pyproject.toml
114 +mypy vllm/prompt_adapter --config-file pyproject.toml
115 mypy tests --config-file pyproject.toml
116
117
tests/lora/test_long_context.py
+4 -5
@@ -92,11 +92,10 @@ def batched_generate(
92 for input in inputs:
93 prompt, sampling_param, lora_req = input
94 # Add requests to the engine and run the engine
95 - llm._validate_and_add_requests(
96 - prompt,
97 - sampling_param,
98 - lora_request=lora_req,
99 - )
95 + llm._validate_and_add_requests(prompt,
96 + sampling_param,
97 + lora_request=lora_req,
98 + prompt_adapter_request=None)
99
100 outputs = llm._run_engine(use_tqdm=True)
101 return [outputs[i].outputs[0].text.strip() for i in range(len(outputs))]
tests/lora/test_lora_manager.py
+164 -162
@@ -127,37 +127,37 @@ def test_lora_model_manager(dist_init, dummy_model):
127 model, 2, 2, 2,
128 LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2))
129 assert all(x is None for x in manager.lora_index_to_id)
130 - assert manager.add_lora(model_lora1)
131 - assert manager.activate_lora(1)
130 + assert manager.add_adapter(model_lora1)
131 + assert manager.activate_adapter(1)
132 assert manager.lora_index_to_id[0] == 1
133 - assert not manager.add_lora(model_lora1)
134 - assert not manager.activate_lora(1)
135 - assert manager.add_lora(model_lora2)
136 - assert manager.activate_lora(2)
133 + assert not manager.add_adapter(model_lora1)
134 + assert not manager.activate_adapter(1)
135 + assert manager.add_adapter(model_lora2)
136 + assert manager.activate_adapter(2)
137 assert manager.lora_index_to_id[0] == 1
138 assert manager.lora_index_to_id[1] == 2
139 - assert not manager.add_lora(model_lora2)
140 - assert not manager.activate_lora(2)
141 - assert manager.add_lora(model_lora3)
139 + assert not manager.add_adapter(model_lora2)
140 + assert not manager.activate_adapter(2)
141 + assert manager.add_adapter(model_lora3)
142 assert manager.lora_index_to_id[0] == 1
143 assert manager.lora_index_to_id[1] == 2
144 with pytest.raises(ValueError):
145 - assert manager.activate_lora(3)
145 + assert manager.activate_adapter(3)
146 assert manager.lora_index_to_id[0] == 1
147 assert manager.lora_index_to_id[1] == 2
148 - assert manager.remove_lora(model_lora2.id)
148 + assert manager.remove_adapter(model_lora2.id)
149 assert manager.lora_index_to_id[1] is None
150 - assert not manager.remove_lora(model_lora2.id)
151 - assert manager.remove_lora(model_lora1.id)
152 - assert not manager.remove_lora(model_lora1.id)
153 - assert manager.add_lora(model_lora1)
150 + assert not manager.remove_adapter(model_lora2.id)
151 + assert manager.remove_adapter(model_lora1.id)
152 + assert not manager.remove_adapter(model_lora1.id)
153 + assert manager.add_adapter(model_lora1)
154 assert manager.lora_index_to_id[0] is None
155 assert manager.lora_index_to_id[1] is None
156 - assert manager.add_lora(model_lora2)
157 - assert manager.activate_lora(3)
156 + assert manager.add_adapter(model_lora2)
157 + assert manager.activate_adapter(3)
158 assert manager.lora_index_to_id[0] == 3
159 assert manager.lora_index_to_id[1] is None
160 - assert manager.activate_lora(2)
160 + assert manager.activate_adapter(2)
161 assert manager.lora_index_to_id[0] == 3
162 assert manager.lora_index_to_id[1] == 2
163
@@ -173,70 +173,70 @@ def test_lora_lru_cache_model_manager(dist_init, dummy_model):
173 model, 2, 2, 2,
174 LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2))
175 assert all(x is None for x in manager.lora_index_to_id)
176 - assert manager.add_lora(model_lora1)
177 - assert manager.activate_lora(1)
176 + assert manager.add_adapter(model_lora1)
177 + assert manager.activate_adapter(1)
178 assert manager.lora_index_to_id[0] == 1
179 - assert not manager.add_lora(model_lora1)
180 - assert not manager.activate_lora(1)
181 - assert manager.add_lora(model_lora2)
182 - assert manager.activate_lora(2)
179 + assert not manager.add_adapter(model_lora1)
180 + assert not manager.activate_adapter(1)
181 + assert manager.add_adapter(model_lora2)
182 + assert manager.activate_adapter(2)
183 assert manager.lora_index_to_id[0] == 1
184 assert manager.lora_index_to_id[1] == 2
185 - assert not manager.add_lora(model_lora2)
186 - assert not manager.activate_lora(2)
187 - assert manager.add_lora(model_lora3)
185 + assert not manager.add_adapter(model_lora2)
186 + assert not manager.activate_adapter(2)
187 + assert manager.add_adapter(model_lora3)
188 assert manager.lora_index_to_id[0] == 1
189 assert manager.lora_index_to_id[1] == 2
190 - assert manager.activate_lora(3)
190 + assert manager.activate_adapter(3)
191 assert manager.lora_index_to_id[0] == 3
192 assert manager.lora_index_to_id[1] == 2
193 - assert manager.remove_lora(model_lora2.id)
193 + assert manager.remove_adapter(model_lora2.id)
194 assert manager.lora_index_to_id[1] is None
195 - assert not manager.remove_lora(model_lora2.id)
196 - assert manager.remove_lora(model_lora1.id)
197 - assert not manager.remove_lora(model_lora1.id)
198 - assert manager.add_lora(model_lora1)
199 - assert manager.activate_lora(1)
195 + assert not manager.remove_adapter(model_lora2.id)
196 + assert manager.remove_adapter(model_lora1.id)
197 + assert not manager.remove_adapter(model_lora1.id)
198 + assert manager.add_adapter(model_lora1)
199 + assert manager.activate_adapter(1)
200 assert manager.lora_index_to_id[0] == 3
201 assert manager.lora_index_to_id[1] == 1
202 - assert manager.add_lora(model_lora2)
203 - assert manager.deactivate_lora(3)
202 + assert manager.add_adapter(model_lora2)
203 + assert manager.deactivate_adapter(3)
204 assert manager.lora_index_to_id[0] is None
205 assert manager.lora_index_to_id[1] == 1
206 - assert manager.activate_lora(2)
206 + assert manager.activate_adapter(2)
207 assert manager.lora_index_to_id[0] == 2
208 assert manager.lora_index_to_id[1] == 1
209 - assert manager.activate_lora(3)
209 + assert manager.activate_adapter(3)
210 assert manager.lora_index_to_id[0] == 2
211 assert manager.lora_index_to_id[1] == 3
212 - assert manager.pin_lora(2)
212 + assert manager.pin_adapter(2)
213 assert manager.lora_index_to_id[0] == 2
214 assert manager.lora_index_to_id[1] == 3
215 - assert manager.activate_lora(1)
215 + assert manager.activate_adapter(1)
216 assert manager.lora_index_to_id[0] == 2
217 assert manager.lora_index_to_id[1] == 1
218 - assert manager.deactivate_lora(2)
218 + assert manager.deactivate_adapter(2)
219 assert manager.lora_index_to_id[0] is None
220 assert manager.lora_index_to_id[1] == 1
221 - assert manager.activate_lora(3)
221 + assert manager.activate_adapter(3)
222 assert manager.lora_index_to_id[0] == 3
223 assert manager.lora_index_to_id[1] == 1
224 - assert manager.pin_lora(3)
225 - assert manager.pin_lora(1)
224 + assert manager.pin_adapter(3)
225 + assert manager.pin_adapter(1)
226 with pytest.raises(RuntimeError):
227 - assert manager.pin_lora(2)
227 + assert manager.pin_adapter(2)
228 assert manager.lora_index_to_id[0] == 3
229 assert manager.lora_index_to_id[1] == 1
230 with pytest.raises(RuntimeError):
231 - assert manager.activate_lora(2)
231 + assert manager.activate_adapter(2)
232
233 - assert manager.deactivate_lora(3)
234 - assert manager.pin_lora(2)
233 + assert manager.deactivate_adapter(3)
234 + assert manager.pin_adapter(2)
235 assert manager.lora_index_to_id[0] == 2
236 assert manager.lora_index_to_id[1] == 1
237 - assert manager.remove_lora(3)
237 + assert manager.remove_adapter(3)
238 with pytest.raises(ValueError):
239 - assert manager.pin_lora(3)
239 + assert manager.pin_adapter(3)
240
241
242 def test_lru_lora_model_manager(dist_init, dummy_model):
@@ -256,168 +256,169 @@ def test_lru_lora_model_manager(dist_init, dummy_model):
256 assert all(x is None for x in manager.lora_index_to_id)
257
258 # Add up to capacity
259 - assert manager.add_lora(model_lora1)
260 - assert manager.add_lora(model_lora2)
261 - assert manager.activate_lora(1)
262 - assert manager.activate_lora(2)
259 + assert manager.add_adapter(model_lora1)
260 + assert manager.add_adapter(model_lora2)
261 + assert manager.activate_adapter(1)
262 + assert manager.activate_adapter(2)
263
264 - assert set(manager.list_loras()) == {1, 2}
264 + assert set(manager.list_adapters()) == {1, 2}
265 assert manager.lora_index_to_id[0] == 1
266 assert manager.lora_index_to_id[1] == 2
267
268 # Add over capacity
269 - assert manager.add_lora(model_lora3)
270 - assert manager.add_lora(model_lora4)
271 - assert manager.activate_lora(3)
272 - assert manager.activate_lora(4)
269 + assert manager.add_adapter(model_lora3)
270 + assert manager.add_adapter(model_lora4)
271 + assert manager.activate_adapter(3)
272 + assert manager.activate_adapter(4)
273
274 - assert set(manager.list_loras()) == {3, 4}
274 + assert set(manager.list_adapters()) == {3, 4}
275 assert manager.lora_index_to_id[0] == 3
276 assert manager.lora_index_to_id[1] == 4
277
278 # Add 3 again to move it to the top and then add 2
279 # should return false since it's in already
280 - assert not manager.add_lora(model_lora3)
281 - assert not manager.activate_lora(3)
282 - assert manager.add_lora(model_lora2)
283 - assert manager.activate_lora(2)
280 + assert not manager.add_adapter(model_lora3)
281 + assert not manager.activate_adapter(3)
282 + assert manager.add_adapter(model_lora2)
283 + assert manager.activate_adapter(2)
284
285 - assert set(manager.list_loras()) == {3, 2}
285 + assert set(manager.list_adapters()) == {3, 2}
286 assert manager.lora_index_to_id[0] == 3
287 assert manager.lora_index_to_id[1] == 2
288
289 # Remove manually
290 - assert manager.remove_lora(3)
291 - assert not manager.remove_lora(3)
290 + assert manager.remove_adapter(3)
291 + assert not manager.remove_adapter(3)
292
293 - assert set(manager.list_loras()) == {2}
293 + assert set(manager.list_adapters()) == {2}
294 assert manager.lora_index_to_id[0] is None
295 assert manager.lora_index_to_id[1] == 2
296
297 - assert manager.add_lora(model_lora3)
298 - assert manager.activate_lora(3)
299 - assert manager.add_lora(model_lora4)
300 - assert manager.activate_lora(4)
297 + assert manager.add_adapter(model_lora3)
298 + assert manager.activate_adapter(3)
299 + assert manager.add_adapter(model_lora4)
300 + assert manager.activate_adapter(4)
301
302 - assert set(manager.list_loras()) == {3, 4}
302 + assert set(manager.list_adapters()) == {3, 4}
303 assert manager.lora_index_to_id[0] == 3
304 assert manager.lora_index_to_id[1] == 4
305
306 - assert manager.remove_oldest_lora()
307 - assert set(manager.list_loras()) == {4}
306 + assert manager.remove_oldest_adapter()
307 + assert set(manager.list_adapters()) == {4}
308 assert manager.lora_index_to_id[0] is None
309 assert manager.lora_index_to_id[1] == 4
310
311 - assert manager.remove_oldest_lora()
312 - assert set(manager.list_loras()) == set()
311 + assert manager.remove_oldest_adapter()
312 + assert set(manager.list_adapters()) == set()
313 assert all(x is None for x in manager.lora_index_to_id)
314
315 - assert not manager.remove_oldest_lora()
316 - assert set(manager.list_loras()) == set()
315 + assert not manager.remove_oldest_adapter()
316 + assert set(manager.list_adapters()) == set()
317 assert all(x is None for x in manager.lora_index_to_id)
318
319 # pinning
320 - assert manager.add_lora(model_lora3)
321 - assert manager.activate_lora(3)
322 - assert manager.add_lora(model_lora4)
323 - assert manager.activate_lora(4)
324 - assert set(manager.list_loras()) == {3, 4}
320 + assert manager.add_adapter(model_lora3)
321 + assert manager.activate_adapter(3)
322 + assert manager.add_adapter(model_lora4)
323 + assert manager.activate_adapter(4)
324 + assert set(manager.list_adapters()) == {3, 4}
325 with pytest.raises(ValueError):
326 - assert manager.pin_lora(1)
327 - assert manager.pin_lora(3)
326 + assert manager.pin_adapter(1)
327 + assert manager.pin_adapter(3)
328 # Remove manually
329 - assert manager.remove_lora(3)
330 - assert not manager.remove_lora(3)
329 + assert manager.remove_adapter(3)
330 + assert not manager.remove_adapter(3)
331
332 - assert set(manager.list_loras()) == {4}
332 + assert set(manager.list_adapters()) == {4}
333 assert manager.lora_index_to_id[0] is None
334 assert manager.lora_index_to_id[1] == 4
335
336 - assert manager.add_lora(model_lora1)
337 - assert manager.pin_lora(1)
338 - assert manager.add_lora(model_lora2)
339 - assert manager.activate_lora(2)
336 + assert manager.add_adapter(model_lora1)
337 + assert manager.pin_adapter(1)
338 + assert manager.add_adapter(model_lora2)
339 + assert manager.activate_adapter(2)
340
341 - assert set(manager.list_loras()) == {1, 2}
341 + assert set(manager.list_adapters()) == {1, 2}
342 assert manager.lora_index_to_id[0] == 1
343 assert manager.lora_index_to_id[1] == 2
344
345 - assert manager.remove_oldest_lora()
346 - assert set(manager.list_loras()) == {1}
345 + assert manager.remove_oldest_adapter()
346 + assert set(manager.list_adapters()) == {1}
347 assert manager.lora_index_to_id[0] == 1
348 assert manager.lora_index_to_id[1] is None
349
350 with pytest.raises(RuntimeError):
351 - assert manager.remove_oldest_lora()
351 + assert manager.remove_oldest_adapter()
352
353 - assert set(manager.list_loras()) == {1}
353 + assert set(manager.list_adapters()) == {1}
354
355
356 -def test_lru_cache_worker_lora_manager(llama_2_7b_model_extra_embeddings,
357 - sql_lora_files):
356 +def test_lru_cache_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
357 + sql_lora_files):
358 lora_config = LoRAConfig(max_lora_rank=8, max_cpu_loras=4, max_loras=4)
359 - worker_lora_manager = LRUCacheWorkerLoRAManager(
359 + worker_adapter_manager = LRUCacheWorkerLoRAManager(
360 4, 2, llama_2_7b_model_extra_embeddings.unpadded_vocab_size -
361 lora_config.lora_extra_vocab_size, lora_config, torch.device("cuda"),
362 EMBEDDING_MODULES, EMBEDDING_PADDING_MODULES)
363 - worker_lora_manager.create_lora_manager(llama_2_7b_model_extra_embeddings)
363 + worker_adapter_manager.create_lora_manager(
364 + llama_2_7b_model_extra_embeddings)
365
366 mapping = LoRAMapping([], [])
366 - worker_lora_manager.set_active_loras([
367 + worker_adapter_manager.set_active_adapters([
368 LoRARequest("1", 1, sql_lora_files),
369 LoRARequest("2", 2, sql_lora_files)
370 ], mapping)
370 - assert worker_lora_manager.list_loras() == {1, 2}
371 - assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
372 - assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
371 + assert worker_adapter_manager.list_adapters() == {1, 2}
372 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[0] == 1
373 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[1] == 2
374
374 - worker_lora_manager.set_active_loras([
375 + worker_adapter_manager.set_active_adapters([
376 LoRARequest("1", 1, sql_lora_files),
377 LoRARequest("3", 3, sql_lora_files),
378 LoRARequest("4", 4, sql_lora_files)
379 ], mapping)
379 - assert worker_lora_manager.list_loras() == {1, 2, 3, 4}
380 - assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
381 - assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
382 - assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 3
383 - assert worker_lora_manager._lora_manager.lora_index_to_id[3] == 4
380 + assert worker_adapter_manager.list_adapters() == {1, 2, 3, 4}
381 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[0] == 1
382 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[1] == 2
383 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[2] == 3
384 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[3] == 4
385
385 - worker_lora_manager.set_active_loras([
386 + worker_adapter_manager.set_active_adapters([
387 LoRARequest("1", 1, sql_lora_files),
388 LoRARequest("2", 2, sql_lora_files),
389 LoRARequest("5", 5, sql_lora_files)
390 ], mapping)
390 - assert worker_lora_manager.list_loras() == {1, 2, 4, 5}
391 - assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
392 - assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
393 - assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 5
394 - assert worker_lora_manager._lora_manager.lora_index_to_id[3] == 4
391 + assert worker_adapter_manager.list_adapters() == {1, 2, 4, 5}
392 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[0] == 1
393 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[1] == 2
394 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[2] == 5
395 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[3] == 4
396
396 - worker_lora_manager.set_active_loras([
397 + worker_adapter_manager.set_active_adapters([
398 LoRARequest("1", 1, sql_lora_files),
399 LoRARequest("1", 1, sql_lora_files),
400 LoRARequest("1", 1, sql_lora_files)
401 ], mapping)
401 - assert worker_lora_manager.list_loras() == {1, 2, 4, 5}
402 - assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
403 - assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
404 - assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 5
405 - assert worker_lora_manager._lora_manager.lora_index_to_id[3] == 4
402 + assert worker_adapter_manager.list_adapters() == {1, 2, 4, 5}
403 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[0] == 1
404 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[1] == 2
405 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[2] == 5
406 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[3] == 4
407
407 - worker_lora_manager.set_active_loras([
408 + worker_adapter_manager.set_active_adapters([
409 LoRARequest("6", 6, sql_lora_files),
410 LoRARequest("7", 7, sql_lora_files),
411 LoRARequest("8", 8, sql_lora_files)
412 ], mapping)
412 - assert worker_lora_manager.list_loras() == {1, 6, 7, 8}
413 - assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
414 - assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 7
415 - assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 8
416 - assert worker_lora_manager._lora_manager.lora_index_to_id[3] == 6
413 + assert worker_adapter_manager.list_adapters() == {1, 6, 7, 8}
414 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[0] == 1
415 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[1] == 7
416 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[2] == 8
417 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[3] == 6
418
419 # Over capacity
420 with pytest.raises(RuntimeError):
420 - worker_lora_manager.set_active_loras([
421 + worker_adapter_manager.set_active_adapters([
422 LoRARequest("10", 10, sql_lora_files),
423 LoRARequest("11", 11, sql_lora_files),
424 LoRARequest("12", 12, sql_lora_files),
@@ -426,68 +427,69 @@ def test_lru_cache_worker_lora_manager(llama_2_7b_model_extra_embeddings,
427 ], mapping)
428
429
429 -def test_worker_lora_manager(llama_2_7b_model_extra_embeddings,
430 - sql_lora_files):
430 +def test_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
431 + sql_lora_files):
432 # Should remove every LoRA not specified in the request.
433 lora_config = LoRAConfig(max_lora_rank=8, max_cpu_loras=4, max_loras=4)
433 - worker_lora_manager = WorkerLoRAManager(
434 + worker_adapter_manager = WorkerLoRAManager(
435 4, 2, llama_2_7b_model_extra_embeddings.unpadded_vocab_size -
436 lora_config.lora_extra_vocab_size, lora_config, torch.device("cuda"),
437 EMBEDDING_MODULES, EMBEDDING_PADDING_MODULES)
437 - worker_lora_manager.create_lora_manager(llama_2_7b_model_extra_embeddings)
438 + worker_adapter_manager.create_lora_manager(
439 + llama_2_7b_model_extra_embeddings)
440
441 mapping = LoRAMapping([], [])
440 - worker_lora_manager.set_active_loras([
442 + worker_adapter_manager.set_active_adapters([
443 LoRARequest("1", 1, sql_lora_files),
444 LoRARequest("2", 2, sql_lora_files)
445 ], mapping)
444 - assert worker_lora_manager.list_loras() == {1, 2}
445 - assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
446 - assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
446 + assert worker_adapter_manager.list_adapters() == {1, 2}
447 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[0] == 1
448 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[1] == 2
449
448 - worker_lora_manager.set_active_loras([
450 + worker_adapter_manager.set_active_adapters([
451 LoRARequest("1", 1, sql_lora_files),
452 LoRARequest("3", 3, sql_lora_files),
453 LoRARequest("4", 4, sql_lora_files)
454 ], mapping)
453 - assert worker_lora_manager.list_loras() == {1, 3, 4}
454 - assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
455 - assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 3
456 - assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 4
455 + assert worker_adapter_manager.list_adapters() == {1, 3, 4}
456 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[0] == 1
457 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[1] == 3
458 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[2] == 4
459
458 - worker_lora_manager.set_active_loras([
460 + worker_adapter_manager.set_active_adapters([
461 LoRARequest("1", 1, sql_lora_files),
462 LoRARequest("2", 2, sql_lora_files),
463 LoRARequest("5", 5, sql_lora_files)
464 ], mapping)
463 - assert worker_lora_manager.list_loras() == {1, 2, 5}
464 - assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
465 - assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
466 - assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 5
465 + assert worker_adapter_manager.list_adapters() == {1, 2, 5}
466 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[0] == 1
467 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[1] == 2
468 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[2] == 5
469
468 - worker_lora_manager.set_active_loras([
470 + worker_adapter_manager.set_active_adapters([
471 LoRARequest("1", 1, sql_lora_files),
472 LoRARequest("1", 1, sql_lora_files),
473 LoRARequest("1", 1, sql_lora_files)
474 ], mapping)
473 - assert worker_lora_manager.list_loras() == {1}
474 - assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
475 - assert worker_lora_manager._lora_manager.lora_index_to_id[1] is None
476 - assert worker_lora_manager._lora_manager.lora_index_to_id[2] is None
475 + assert worker_adapter_manager.list_adapters() == {1}
476 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[0] == 1
477 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[1] is None
478 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[2] is None
479
478 - worker_lora_manager.set_active_loras([
480 + worker_adapter_manager.set_active_adapters([
481 LoRARequest("6", 6, sql_lora_files),
482 LoRARequest("7", 7, sql_lora_files),
483 LoRARequest("8", 8, sql_lora_files)
484 ], mapping)
483 - assert worker_lora_manager.list_loras() == {6, 7, 8}
484 - assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 8
485 - assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 6
486 - assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 7
485 + assert worker_adapter_manager.list_adapters() == {6, 7, 8}
486 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[0] == 8
487 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[1] == 6
488 + assert worker_adapter_manager._adapter_manager.lora_index_to_id[2] == 7
489
490 # Over capacity
491 with pytest.raises(RuntimeError):
490 - worker_lora_manager.set_active_loras([
492 + worker_adapter_manager.set_active_adapters([
493 LoRARequest("10", 10, sql_lora_files),
494 LoRARequest("11", 11, sql_lora_files),
495 LoRARequest("12", 12, sql_lora_files),
@@ -525,8 +527,8 @@ def test_packed_loras(dist_init, dummy_model_gate_up):
527
528 assert isinstance(model.get_submodule("gate_up_proj"),
529 MergedColumnParallelLinearWithLoRA)
528 - assert manager.add_lora(model_lora)
529 - assert manager.add_lora(model_lora1)
530 + assert manager.add_adapter(model_lora)
531 + assert manager.add_adapter(model_lora1)
532
533 packed_lora = model_lora.get_lora("gate_up_proj")
534 assert packed_lora and isinstance(packed_lora, PackedLoRALayerWeights)
tests/prompt_adapter/test_bloom.py new
+45
@@ -0,0 +1,45 @@
1 +import pytest
2 +
3 +import vllm
4 +from vllm.prompt_adapter.request import PromptAdapterRequest
5 +
6 +MODEL_PATH = "bigscience/bloomz-560m"
7 +PA_PATH = 'stevhliu/bloomz-560m_PROMPT_TUNING_CAUSAL_LM'
8 +
9 +
10 +def do_sample(llm, pa_name: str, pa_id: int):
11 +
12 + prompts = [
13 + "Tweet text : @nationalgridus I have no water and the bill is \
14 + current and paid. Can you do something about this? Label : ",
15 + "Tweet text : @nationalgridus Looks good thanks! Label : "
16 + ]
17 + sampling_params = vllm.SamplingParams(temperature=0.0,
18 + max_tokens=3,
19 + stop_token_ids=[3])
20 +
21 + outputs = llm.generate(prompts,
22 + sampling_params,
23 + prompt_adapter_request=PromptAdapterRequest(
24 + pa_name, pa_id, PA_PATH, 8) if pa_id else None)
25 +
26 + # Print the outputs.
27 + generated_texts = []
28 + for output in outputs:
29 + prompt = output.prompt
30 + generated_text = output.outputs[0].text.strip()
31 + generated_texts.append(generated_text)
32 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
33 + return generated_texts
34 +
35 +
36 +@pytest.mark.parametrize("enforce_eager", [True, False])
37 +def test_twitter_prompt_adapter(enforce_eager: bool):
38 + llm = vllm.LLM(MODEL_PATH,
39 + enforce_eager=enforce_eager,
40 + enable_prompt_adapter=True,
41 + max_prompt_adapter_token=8)
42 +
43 + expected_output = ['complaint', 'no complaint']
44 +
45 + assert do_sample(llm, "twitter_pa", pa_id=1) == expected_output
tests/prompt_adapter/test_multi_adapter_inference.py new
+53
@@ -0,0 +1,53 @@
1 +from vllm import EngineArgs, LLMEngine, SamplingParams
2 +from vllm.prompt_adapter.request import PromptAdapterRequest
3 +
4 +MODEL_PATH = "bigscience/bloomz-560m"
5 +pa_path = 'stevhliu/bloomz-560m_PROMPT_TUNING_CAUSAL_LM'
6 +pa_path2 = 'swapnilbp/angry_tweet_ptune'
7 +
8 +
9 +def do_sample(engine):
10 +
11 + prompts = [
12 + ("Tweet text: I have complaints! Label: ",
13 + SamplingParams(temperature=0.0, max_tokens=3, stop_token_ids=[3]),
14 + PromptAdapterRequest("hate_speech", 1, pa_path2, 8)),
15 + ("Tweet text: I have no problems Label: ",
16 + SamplingParams(temperature=0.0, max_tokens=3, stop_token_ids=[3]),
17 + PromptAdapterRequest("hate_speech2", 2, pa_path2, 8)),
18 + ("Tweet text: I have complaints! Label: ",
19 + SamplingParams(temperature=0.0, max_tokens=3), None),
20 + ("Tweet text: I have no problems Label: ",
21 + SamplingParams(temperature=0.0, max_tokens=3, stop_token_ids=[3]),
22 + PromptAdapterRequest("complain", 3, pa_path, 8)),
23 + ]
24 +
25 + request_id = 0
26 + results = set()
27 + while prompts or engine.has_unfinished_requests():
28 + if prompts:
29 + prompt, sampling_params, pa_request = prompts.pop(0)
30 + engine.add_request(str(request_id),
31 + prompt,
32 + sampling_params,
33 + prompt_adapter_request=pa_request)
34 + request_id += 1
35 +
36 + request_outputs = engine.step()
37 +
38 + for request_output in request_outputs:
39 + if request_output.finished:
40 + results.add(request_output.outputs[0].text)
41 + return results
42 +
43 +
44 +def test_multi_prompt_adapters():
45 + engine_args = EngineArgs(model=MODEL_PATH,
46 + max_prompt_adapters=3,
47 + enable_prompt_adapter=True,
48 + max_prompt_adapter_token=8)
49 + engine = LLMEngine.from_engine_args(engine_args)
50 + expected_output = {
51 + ' quot;I', 'hate speech', 'no complaint', 'not hate speech'
52 + }
53 + assert do_sample(engine) == expected_output
tests/prompt_adapter/test_pa_lora.py new
+61
@@ -0,0 +1,61 @@
1 +from huggingface_hub import snapshot_download
2 +
3 +from vllm import EngineArgs, LLMEngine, SamplingParams
4 +from vllm.lora.request import LoRARequest
5 +from vllm.prompt_adapter.request import PromptAdapterRequest
6 +
7 +MODEL_PATH = "meta-llama/Llama-2-7b-hf"
8 +pa_path = snapshot_download(repo_id="swapnilbp/llama_tweet_ptune")
9 +lora_path = snapshot_download(repo_id="yard1/llama-2-7b-sql-lora-test")
10 +
11 +
12 +def do_sample(engine):
13 +
14 + prompt_text = "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]" # noqa: E501
15 +
16 + # first prompt with a prompt adapter and second without adapter
17 + prompts = [
18 + (prompt_text,
19 + SamplingParams(temperature=0.0, max_tokens=100,
20 + stop=["[/assistant]"]),
21 + PromptAdapterRequest("hate_speech", 1, pa_path,
22 + 8), LoRARequest("sql_test", 1, lora_path)),
23 + (prompt_text,
24 + SamplingParams(temperature=0.0, max_tokens=100,
25 + stop=["[/assistant]"]), None,
26 + LoRARequest("sql_test", 1, lora_path)),
27 + ]
28 +
29 + request_id = 0
30 + results = set()
31 + while prompts or engine.has_unfinished_requests():
32 + if prompts:
33 + prompt, sampling_params, pa_request, lora_request = prompts.pop(0)
34 + engine.add_request(str(request_id),
35 + prompt,
36 + sampling_params,
37 + prompt_adapter_request=pa_request,
38 + lora_request=lora_request)
39 + request_id += 1
40 +
41 + request_outputs = engine.step()
42 +
43 + for request_output in request_outputs:
44 + if request_output.finished:
45 + results.add(request_output.outputs[0].text)
46 + return results
47 +
48 +
49 +def test_lora_prompt_adapter():
50 + engine_args = EngineArgs(model=MODEL_PATH,
51 + enable_prompt_adapter=True,
52 + enable_lora=True,
53 + max_num_seqs=60,
54 + max_prompt_adapter_token=8)
55 + engine = LLMEngine.from_engine_args(engine_args)
56 + result = do_sample(engine)
57 +
58 + expected_output = {
59 + " SELECT icao FROM table_name_74 WHERE airport = 'lilongwe international airport' " # noqa: E501
60 + }
61 + assert result == expected_output
tests/spec_decode/e2e/conftest.py
+2
@@ -13,6 +13,7 @@ from vllm.lora.request import LoRARequest
13 from vllm.model_executor.utils import set_random_seed
14 from vllm.multimodal import MultiModalDataDict
15 from vllm.outputs import RequestOutput
16 +from vllm.prompt_adapter.request import PromptAdapterRequest
17 from vllm.sampling_params import SamplingParams
18 from vllm.sequence import Logprob
19 from vllm.usage.usage_lib import UsageContext
@@ -92,6 +93,7 @@ class AsyncLLM:
93 use_tqdm: bool = True,
94 lora_request: Optional[LoRARequest] = None,
95 multi_modal_data: Optional[MultiModalDataDict] = None,
96 + prompt_adapter_request: Optional[PromptAdapterRequest] = None
97 ) -> List[RequestOutput]:
98
99 if prompts is None:
tests/worker/test_model_runner.py
+1
@@ -23,6 +23,7 @@ def _create_model_runner(model: str, *args, **kwargs) -> ModelRunner:
23 cache_config=engine_config.cache_config,
24 load_config=engine_config.load_config,
25 lora_config=engine_config.lora_config,
26 + prompt_adapter_config=engine_config.prompt_adapter_config,
27 is_driver_worker=True,
28 )
29 return model_runner
vllm/adapter_commons/__init__.py
vllm/adapter_commons/layers.py new
+14
@@ -0,0 +1,14 @@
1 +from dataclasses import dataclass
2 +from typing import Tuple
3 +
4 +
5 +@dataclass
6 +class AdapterMapping:
7 + # Per every token in input_ids:
8 + index_mapping: Tuple[int, ...]
9 + # Per sampled token:
10 + prompt_mapping: Tuple[int, ...]
11 +
12 + def __post_init__(self):
13 + self.index_mapping = tuple(self.index_mapping)
14 + self.prompt_mapping = tuple(self.prompt_mapping)
\ No newline at end of file
vllm/adapter_commons/models.py new
+104
@@ -0,0 +1,104 @@
1 +from abc import ABC, abstractmethod
2 +from typing import Any, Callable, Dict, Hashable, Optional, TypeVar
3 +
4 +from torch import nn
5 +
6 +from vllm.logger import init_logger
7 +from vllm.utils import LRUCache
8 +
9 +logger = init_logger(__name__)
10 +
11 +
12 +class AdapterModel(ABC):
13 +
14 + def __init__(self, model_id=None):
15 + self.id = model_id
16 +
17 + @abstractmethod
18 + def from_local_checkpoint(cls, model_dir, model_id=None, **kwargs):
19 + # Common initialization code
20 + # Load weights or embeddings from local checkpoint
21 + raise NotImplementedError("Subclasses must implement this method.")
22 +
23 +
24 +T = TypeVar('T')
25 +
26 +
27 +class AdapterLRUCache(LRUCache[T]):
28 +
29 + def __init__(self, capacity: int, deactivate_fn: Callable[[Hashable],
30 + None]):
31 + super().__init__(capacity)
32 + self.deactivate_fn = deactivate_fn
33 +
34 + def _on_remove(self, key: Hashable, value: T):
35 + logger.debug("Removing adapter int id: %d", key)
36 + self.deactivate_fn(key)
37 + return super()._on_remove(key, value)
38 +
39 +
40 +class AdapterModelManager(ABC):
41 +
42 + def __init__(
43 + self,
44 + model: nn.Module,
45 + ):
46 + """Create a AdapterModelManager and adapter for a given model.
47 + Args:
48 + model: the model to be adapted.
49 + """
50 + self.model: nn.Module = model
51 + self._registered_adapters: Dict[int, Any] = {}
52 + # Dict instead of a Set for compatibility with LRUCache.
53 + self._active_adapters: Dict[int, None] = {}
54 + self.adapter_type = 'Adapter'
55 + self._last_mapping = None
56 +
57 + def __len__(self) -> int:
58 + return len(self._registered_adapters)
59 +
60 + @property
61 + @abstractmethod
62 + def adapter_slots(self):
63 + ...
64 +
65 + @property
66 + @abstractmethod
67 + def capacity(self):
68 + ...
69 +
70 + @abstractmethod
71 + def activate_adapter(self, adapter_id: int) -> bool:
72 + ...
73 +
74 + @abstractmethod
75 + def deactivate_adapter(self, adapter_id: int) -> bool:
76 + ...
77 +
78 + @abstractmethod
79 + def add_adapter(self, adapter: Any) -> bool:
80 + ...
81 +
82 + @abstractmethod
83 + def set_adapter_mapping(self, mapping: Any) -> None:
84 + ...
85 +
86 + @abstractmethod
87 + def remove_adapter(self, adapter_id: int) -> bool:
88 + ...
89 +
90 + @abstractmethod
91 + def remove_all_adapters(self):
92 + ...
93 +
94 + @abstractmethod
95 + def get_adapter(self, adapter_id: int) -> Optional[Any]:
96 + ...
97 +
98 + @abstractmethod
99 + def list_adapters(self) -> Dict[int, Any]:
100 + ...
101 +
102 + @abstractmethod
103 + def pin_adapter(self, adapter_id: int) -> bool:
104 + ...
vllm/adapter_commons/request.py new
+25
@@ -0,0 +1,25 @@
1 +from abc import abstractmethod
2 +from dataclasses import dataclass
3 +
4 +
5 +@dataclass
6 +class AdapterRequest:
7 + """
8 + Base class for adapter requests.
9 + """
10 +
11 + @property
12 + @abstractmethod
13 + def adapter_id(self):
14 + ...
15 +
16 + def __post_init__(self):
17 + if self.adapter_id < 1:
18 + raise ValueError(f"id must be > 0, got {self.adapter_id}")
19 +
20 + def __eq__(self, value: object) -> bool:
21 + return isinstance(
22 + value, self.__class__) and self.adapter_id == value.adapter_id
23 +
24 + def __hash__(self) -> int:
25 + return hash(self.adapter_id)
vllm/adapter_commons/utils.py new
+90
@@ -0,0 +1,90 @@
1 +from typing import Any, Callable, Dict, Optional, Set
2 +
3 +
4 +## model functions
5 +def deactivate_adapter(adapter_id: int, active_adapters: Dict[int, None],
6 + deactivate_func: Callable) -> bool:
7 + if adapter_id in active_adapters:
8 + deactivate_func(adapter_id)
9 + active_adapters.pop(adapter_id)
10 + return True
11 + return False
12 +
13 +
14 +def add_adapter(adapter: Any, registered_adapters: Dict[int, Any],
15 + capacity: int, add_func: Callable) -> bool:
16 + if adapter.id not in registered_adapters:
17 + if len(registered_adapters) >= capacity:
18 + raise RuntimeError('No free adapter slots.')
19 + add_func(adapter)
20 + registered_adapters[adapter.id] = adapter
21 + return True
22 + return False
23 +
24 +
25 +def set_adapter_mapping(mapping: Any, last_mapping: Any,
26 + set_mapping_func: Callable) -> Any:
27 + if last_mapping != mapping:
28 + set_mapping_func(mapping)
29 + return mapping
30 + return last_mapping
31 +
32 +
33 +def remove_adapter(adapter_id: int, registered_adapters: Dict[int, Any],
34 + deactivate_func: Callable) -> bool:
35 + deactivate_func(adapter_id)
36 + return bool(registered_adapters.pop(adapter_id, None))
37 +
38 +
39 +def list_adapters(registered_adapters: Dict[int, Any]) -> Dict[int, Any]:
40 + return dict(registered_adapters)
41 +
42 +
43 +def get_adapter(adapter_id: int,
44 + registered_adapters: Dict[int, Any]) -> Optional[Any]:
45 + return registered_adapters.get(adapter_id, None)
46 +
47 +
48 +## worker functions
49 +def set_active_adapters_worker(requests: Set[Any], mapping: Optional[Any],
50 + apply_adapters_func,
51 + set_adapter_mapping_func) -> None:
52 + apply_adapters_func(requests)
53 + set_adapter_mapping_func(mapping)
54 +
55 +
56 +def add_adapter_worker(adapter_request: Any, list_adapters_func,
57 + load_adapter_func, add_adapter_func,
58 + activate_adapter_func) -> bool:
59 + if adapter_request.adapter_id in list_adapters_func():
60 + return False
61 + loaded_adapter = load_adapter_func(adapter_request)
62 + loaded = add_adapter_func(loaded_adapter)
63 + activate_adapter_func(loaded_adapter.id)
64 + return loaded
65 +
66 +
67 +def apply_adapters_worker(adapter_requests: Set[Any], list_adapters_func,
68 + adapter_slots: int, remove_adapter_func,
69 + add_adapter_func) -> None:
70 + models_that_exist = list_adapters_func()
71 + models_map = {
72 + adapter_request.adapter_id: adapter_request
73 + for adapter_request in adapter_requests if adapter_request
74 + }
75 + if len(models_map) > adapter_slots:
76 + raise RuntimeError(
77 + f"Number of requested models ({len(models_map)}) is greater "
78 + f"than the number of GPU model slots "
79 + f"({adapter_slots}).")
80 + new_models = set(models_map)
81 + models_to_add = new_models - models_that_exist
82 + models_to_remove = models_that_exist - new_models
83 + for adapter_id in models_to_remove:
84 + remove_adapter_func(adapter_id)
85 + for adapter_id in models_to_add:
86 + add_adapter_func(models_map[adapter_id])
87 +
88 +
89 +def list_adapters_worker(adapter_manager_list_adapters_func) -> Set[int]:
90 + return set(adapter_manager_list_adapters_func())
vllm/adapter_commons/worker_manager.py new
+36
@@ -0,0 +1,36 @@
1 +from abc import ABC, abstractmethod
2 +from typing import Any, Optional, Set
3 +
4 +import torch
5 +
6 +
7 +class AbstractWorkerManager(ABC):
8 +
9 + def __init__(self, device: torch.device):
10 + self.device = device
11 +
12 + @property
13 + @abstractmethod
14 + def is_enabled(self) -> bool:
15 + ...
16 +
17 + @abstractmethod
18 + def set_active_adapters(self, requests: Set[Any],
19 + mapping: Optional[Any]) -> None:
20 + ...
21 +
22 + @abstractmethod
23 + def add_adapter(self, adapter_request: Any) -> bool:
24 + ...
25 +
26 + @abstractmethod
27 + def remove_adapter(self, adapter_id: int) -> bool:
28 + ...
29 +
30 + @abstractmethod
31 + def remove_all_adapters(self):
32 + ...
33 +
34 + @abstractmethod
35 + def list_adapters(self) -> Set[int]:
36 + ...
vllm/config.py
+37
@@ -1285,6 +1285,39 @@ class LoRAConfig:
1285 raise ValueError("LoRA is not supported with chunked prefill yet.")
1286
1287
1288 +@dataclass
1289 +class PromptAdapterConfig:
1290 + max_prompt_adapters: int
1291 + max_prompt_adapter_token: int
1292 + max_cpu_prompt_adapters: Optional[int] = None
1293 + prompt_adapter_dtype: Optional[torch.dtype] = None
1294 +
1295 + def __post_init__(self):
1296 + library_name = 'peft'
1297 + try:
1298 + __import__(library_name)
1299 + except ImportError as e:
1300 + raise ImportError(
1301 + f"'{library_name}' is not installed for prompt adapter support."
1302 + f"Please install it using 'pip install {library_name}'."
1303 + ) from e
1304 +
1305 + if self.max_prompt_adapters < 1:
1306 + raise ValueError(f"max_prompt_adapters "
1307 + f"({self.max_prompt_adapters}) must be >= 1.")
1308 + if self.max_prompt_adapter_token == 0:
1309 + raise ValueError("max_prompt_adapter_token must be set.")
1310 + if self.max_cpu_prompt_adapters is None:
1311 + self.max_cpu_prompt_adapters = self.max_prompt_adapters
1312 +
1313 + def verify_with_model_config(self, model_config: ModelConfig):
1314 + if self.prompt_adapter_dtype in (None, "auto"):
1315 + self.prompt_adapter_dtype = model_config.dtype
1316 + elif isinstance(self.prompt_adapter_dtype, str):
1317 + self.prompt_adapter_dtype = getattr(torch,
1318 + self.prompt_adapter_dtype)
1319 +
1320 +
1321 @dataclass
1322 class MultiModalConfig:
1323 """Configs the input data format and how models should run for
@@ -1518,6 +1551,7 @@ class EngineConfig:
1551 speculative_config: Optional[SpeculativeConfig]
1552 decoding_config: Optional[DecodingConfig]
1553 observability_config: Optional[ObservabilityConfig]
1554 + prompt_adapter_config: Optional[PromptAdapterConfig]
1555
1556 def __post_init__(self):
1557 """Verify configs are valid & consistent with each other.
@@ -1529,6 +1563,9 @@ class EngineConfig:
1563 self.lora_config.verify_with_model_config(self.model_config)
1564 self.lora_config.verify_with_scheduler_config(
1565 self.scheduler_config)
1566 + if self.prompt_adapter_config:
1567 + self.prompt_adapter_config.verify_with_model_config(
1568 + self.model_config)
1569
1570 def to_dict(self):
1571 """Return the configs as a dictionary, for use in **kwargs.
vllm/core/scheduler.py
+12
@@ -11,6 +11,7 @@ from vllm.core.interfaces import AllocStatus, BlockSpaceManager
11 from vllm.core.policy import Policy, PolicyFactory
12 from vllm.logger import init_logger
13 from vllm.lora.request import LoRARequest
14 +from vllm.prompt_adapter.request import PromptAdapterRequest
15 from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
16 SequenceGroupMetadata, SequenceStatus)
17
@@ -139,6 +140,8 @@ class SchedulerOutputs:
140 if self.num_loras > 0:
141 self._sort_by_lora_ids()
142
143 + self.num_prompt_adapters: int = len(self.prompt_adapter_requests)
144 +
145 def is_empty(self) -> bool:
146 # NOTE: We do not consider the ignored sequence groups.
147 return (not self.scheduled_seq_groups and not self.blocks_to_swap_in
@@ -157,6 +160,14 @@ class SchedulerOutputs:
160 if g.seq_group.lora_request is not None
161 }
162
163 + @property
164 + def prompt_adapter_requests(self) -> Set[PromptAdapterRequest]:
165 + return {
166 + g.seq_group.prompt_adapter_request
167 + for g in self.scheduled_seq_groups
168 + if g.seq_group.prompt_adapter_request is not None
169 + }
170 +
171
172 @dataclass
173 class SchedulerRunningOutputs:
@@ -1024,6 +1035,7 @@ class Scheduler:
1035 # `multi_modal_data` will be None.
1036 multi_modal_data=seq_group.multi_modal_data
1037 if scheduler_outputs.num_prefill_groups > 0 else None,
1038 + prompt_adapter_request=seq_group.prompt_adapter_request,
1039 )
1040 seq_group_metadata_list.append(seq_group_metadata)
1041
vllm/engine/arg_utils.py
+22 -2
@@ -7,8 +7,8 @@ from typing import List, Optional, Tuple, Union
7 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
8 EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
9 MultiModalConfig, ObservabilityConfig, ParallelConfig,
10 - SchedulerConfig, SpeculativeConfig,
11 - TokenizerPoolConfig)
10 + PromptAdapterConfig, SchedulerConfig,
11 + SpeculativeConfig, TokenizerPoolConfig)
12 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
13 from vllm.utils import FlexibleArgumentParser
14
@@ -66,6 +66,9 @@ class EngineArgs:
66 enable_lora: bool = False
67 max_loras: int = 1
68 max_lora_rank: int = 16
69 + enable_prompt_adapter: bool = False
70 + max_prompt_adapters: int = 1
71 + max_prompt_adapter_token: int = 0
72 fully_sharded_loras: bool = False
73 lora_extra_vocab_size: int = 256
74 long_lora_scaling_factors: Optional[Tuple[float]] = None
@@ -449,6 +452,17 @@ class EngineArgs:
452 'Enabling this will use the fully sharded layers. '
453 'At high sequence length, max rank or '
454 'tensor parallel size, this is likely faster.'))
455 + parser.add_argument('--enable-prompt-adapter',
456 + action='store_true',
457 + help='If True, enable handling of PromptAdapters.')
458 + parser.add_argument('--max-prompt-adapters',
459 + type=int,
460 + default=EngineArgs.max_prompt_adapters,
461 + help='Max number of PromptAdapters in a batch.')
462 + parser.add_argument('--max-prompt-adapter-token',
463 + type=int,
464 + default=EngineArgs.max_prompt_adapter_token,
465 + help='Max number of PromptAdapters tokens')
466 parser.add_argument("--device",
467 type=str,
468 default=EngineArgs.device,
@@ -726,6 +740,11 @@ class EngineArgs:
740 model_loader_extra_config=self.model_loader_extra_config,
741 )
742
743 + prompt_adapter_config = PromptAdapterConfig(
744 + max_prompt_adapters=self.max_prompt_adapters,
745 + max_prompt_adapter_token=self.max_prompt_adapter_token) \
746 + if self.enable_prompt_adapter else None
747 +
748 decoding_config = DecodingConfig(
749 guided_decoding_backend=self.guided_decoding_backend)
750
@@ -751,6 +770,7 @@ class EngineArgs:
770 load_config=load_config,
771 decoding_config=decoding_config,
772 observability_config=observability_config,
773 + prompt_adapter_config=prompt_adapter_config,
774 )
775
776
vllm/engine/async_llm_engine.py
+29 -9
@@ -18,6 +18,7 @@ from vllm.logger import init_logger
18 from vllm.lora.request import LoRARequest
19 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
20 from vllm.pooling_params import PoolingParams
21 +from vllm.prompt_adapter.request import PromptAdapterRequest
22 from vllm.sampling_params import SamplingParams
23 from vllm.sequence import ExecuteModelRequest, SamplerOutput
24 from vllm.usage.usage_lib import UsageContext
@@ -264,6 +265,7 @@ class _AsyncLLMEngine(LLMEngine):
265 request_id: str,
266 inputs: PromptInputs,
267 lora_request: Optional[LoRARequest] = None,
268 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
269 ) -> LLMInputs:
270 if isinstance(inputs, str):
271 inputs = {"prompt": inputs}
@@ -279,6 +281,12 @@ class _AsyncLLMEngine(LLMEngine):
281 else:
282 prompt_token_ids = inputs["prompt_token_ids"]
283
284 + if prompt_adapter_request:
285 + prompt_token_ids = [
286 + 0
287 + ] * prompt_adapter_request.prompt_adapter_num_virtual_tokens + \
288 + prompt_token_ids
289 +
290 llm_inputs = LLMInputs(prompt_token_ids=prompt_token_ids,
291 prompt=inputs.get("prompt"),
292 multi_modal_data=inputs.get("multi_modal_data"))
@@ -286,13 +294,14 @@ class _AsyncLLMEngine(LLMEngine):
294 return self.input_processor(llm_inputs)
295
296 async def add_request_async(
289 - self,
290 - request_id: str,
291 - inputs: PromptInputs,
292 - params: Union[SamplingParams, PoolingParams],
293 - arrival_time: Optional[float] = None,
294 - lora_request: Optional[LoRARequest] = None,
295 - trace_headers: Optional[Dict[str, str]] = None,
297 + self,
298 + request_id: str,
299 + inputs: PromptInputs,
300 + params: Union[SamplingParams, PoolingParams],
301 + arrival_time: Optional[float] = None,
302 + lora_request: Optional[LoRARequest] = None,
303 + trace_headers: Optional[Dict[str, str]] = None,
304 + prompt_adapter_request: Optional[PromptAdapterRequest] = None
305 ) -> None:
306 if lora_request is not None and not self.lora_config:
307 raise ValueError(f"Got lora_request {lora_request} but LoRA is "
@@ -301,7 +310,10 @@ class _AsyncLLMEngine(LLMEngine):
310 arrival_time = time.time()
311
312 processed_inputs = await self.process_model_inputs_async(
304 - request_id=request_id, inputs=inputs, lora_request=lora_request)
313 + request_id=request_id,
314 + inputs=inputs,
315 + lora_request=lora_request,
316 + prompt_adapter_request=prompt_adapter_request)
317
318 self._add_processed_request(
319 request_id=request_id,
@@ -309,6 +321,7 @@ class _AsyncLLMEngine(LLMEngine):
321 params=params,
322 arrival_time=arrival_time,
323 lora_request=lora_request,
324 + prompt_adapter_request=prompt_adapter_request,
325 trace_headers=trace_headers,
326 )
327
@@ -627,6 +640,7 @@ class AsyncLLMEngine:
640 arrival_time: Optional[float] = None,
641 lora_request: Optional[LoRARequest] = None,
642 trace_headers: Optional[Dict[str, str]] = None,
643 + prompt_adapter_request: Optional[PromptAdapterRequest] = None
644 ) -> AsyncStream:
645 if self.log_requests:
646 if isinstance(inputs, str):
@@ -669,7 +683,7 @@ class AsyncLLMEngine:
683 arrival_time=arrival_time,
684 lora_request=lora_request,
685 trace_headers=trace_headers,
672 - )
686 + prompt_adapter_request=prompt_adapter_request)
687
688 return stream
689
@@ -680,6 +694,7 @@ class AsyncLLMEngine:
694 request_id: str,
695 lora_request: Optional[LoRARequest] = None,
696 trace_headers: Optional[Dict[str, str]] = None,
697 + prompt_adapter_request: Optional[PromptAdapterRequest] = None
698 ) -> AsyncIterator[RequestOutput]:
699 """Generate outputs for a request.
700
@@ -695,6 +710,8 @@ class AsyncLLMEngine:
710 request_id: The unique id of the request.
711 lora_request: LoRA request to use for generation, if any.
712 trace_headers: OpenTelemetry trace headers.
713 + prompt_adapter_request: Prompt Adapter request to use
714 + for generation, if any.
715
716 Yields:
717 The output `RequestOutput` objects from the LLMEngine
@@ -749,6 +766,7 @@ class AsyncLLMEngine:
766 sampling_params,
767 lora_request=lora_request,
768 trace_headers=trace_headers,
769 + prompt_adapter_request=prompt_adapter_request,
770 ):
771 yield LLMEngine.validate_output(output, RequestOutput)
772
@@ -837,6 +855,7 @@ class AsyncLLMEngine:
855 *,
856 lora_request: Optional[LoRARequest] = None,
857 trace_headers: Optional[Dict[str, str]] = None,
858 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
859 ) -> AsyncIterator[Union[RequestOutput, EmbeddingRequestOutput]]:
860 """Common logic to process requests with SamplingParams or
861 PoolingParams."""
@@ -849,6 +868,7 @@ class AsyncLLMEngine:
868 arrival_time=arrival_time,
869 lora_request=lora_request,
870 trace_headers=trace_headers,
871 + prompt_adapter_request=prompt_adapter_request,
872 )
873
874 try:
vllm/engine/llm_engine.py
+50 -15
@@ -8,7 +8,8 @@ from transformers import PreTrainedTokenizer
8
9 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
10 LoRAConfig, ModelConfig, MultiModalConfig,
11 - ObservabilityConfig, ParallelConfig, SchedulerConfig,
11 + ObservabilityConfig, ParallelConfig,
12 + PromptAdapterConfig, SchedulerConfig,
13 SpeculativeConfig)
14 from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler,
15 SchedulerOutputs)
@@ -27,6 +28,7 @@ from vllm.lora.request import LoRARequest
28 from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
29 RequestOutputFactory)
30 from vllm.pooling_params import PoolingParams
31 +from vllm.prompt_adapter.request import PromptAdapterRequest
32 from vllm.sampling_params import SamplingParams
33 from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
34 PoolerOutput, SamplerOutput, Sequence,
@@ -93,6 +95,8 @@ class LLMEngine:
95 decoding.
96 executor_class: The model executor class for managing distributed
97 execution.
98 + prompt_adapter_config (Optional): The configuration related to serving
99 + prompt adapters.
100 log_stats: Whether to log statistics.
101 usage_context: Specified entry point, used for usage info collection.
102 """
@@ -161,6 +165,7 @@ class LLMEngine:
165 speculative_config: Optional[SpeculativeConfig],
166 decoding_config: Optional[DecodingConfig],
167 observability_config: Optional[ObservabilityConfig],
168 + prompt_adapter_config: Optional[PromptAdapterConfig],
169 executor_class: Type[ExecutorBase],
170 log_stats: bool,
171 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
@@ -222,6 +227,7 @@ class LLMEngine:
227 self.speculative_config = speculative_config
228 self.load_config = load_config
229 self.decoding_config = decoding_config or DecodingConfig()
230 + self.prompt_adapter_config = prompt_adapter_config
231 self.observability_config = observability_config or ObservabilityConfig(
232 )
233 self.log_stats = log_stats
@@ -250,6 +256,7 @@ class LLMEngine:
256 multimodal_config=multimodal_config,
257 speculative_config=speculative_config,
258 load_config=load_config,
259 + prompt_adapter_config=prompt_adapter_config,
260 )
261
262 if not self.model_config.embedding_mode:
@@ -282,6 +289,8 @@ class LLMEngine:
289 # Feature flags
290 "enable_lora":
291 bool(lora_config),
292 + "enable_prompt_adapter":
293 + bool(prompt_adapter_config),
294 "enable_prefix_caching":
295 cache_config.enable_prefix_caching,
296 "enforce_eager":
@@ -376,7 +385,6 @@ class LLMEngine:
385 engine_config = engine_args.create_engine_config()
386 distributed_executor_backend = (
387 engine_config.parallel_config.distributed_executor_backend)
379 -
388 # Initialize the cluster and specify the executor class.
389 if engine_config.device_config.device_type == "neuron":
390 from vllm.executor.neuron_executor import NeuronExecutor
@@ -409,7 +417,6 @@ class LLMEngine:
417 else:
418 from vllm.executor.gpu_executor import GPUExecutor
419 executor_class = GPUExecutor
412 -
420 # Create the LLM engine.
421 engine = cls(
422 **engine_config.to_dict(),
@@ -470,6 +477,9 @@ class LLMEngine:
477 self.lora_config.verify_with_model_config(self.model_config)
478 self.lora_config.verify_with_scheduler_config(
479 self.scheduler_config)
480 + if self.prompt_adapter_config:
481 + self.prompt_adapter_config.verify_with_model_config(
482 + self.model_config)
483
484 def _get_eos_token_id(
485 self, lora_request: Optional[LoRARequest]) -> Optional[int]:
@@ -487,6 +497,7 @@ class LLMEngine:
497 params: Union[SamplingParams, PoolingParams],
498 arrival_time: float,
499 lora_request: Optional[LoRARequest],
500 + prompt_adapter_request: Optional[PromptAdapterRequest],
501 trace_headers: Optional[Dict[str, str]] = None,
502 ) -> None:
503 # Create the sequences.
@@ -495,7 +506,7 @@ class LLMEngine:
506 eos_token_id = self._get_eos_token_id(lora_request)
507
508 seq = Sequence(seq_id, processed_inputs, block_size, eos_token_id,
498 - lora_request)
509 + lora_request, prompt_adapter_request)
510
511 # Create a SequenceGroup based on SamplingParams or PoolingParams
512 if isinstance(params, SamplingParams):
@@ -506,7 +517,7 @@ class LLMEngine:
517 arrival_time=arrival_time,
518 lora_request=lora_request,
519 trace_headers=trace_headers,
509 - )
520 + prompt_adapter_request=prompt_adapter_request)
521 elif isinstance(params, PoolingParams):
522 seq_group = self._create_sequence_group_with_pooling(
523 request_id,
@@ -514,7 +525,7 @@ class LLMEngine:
525 params,
526 arrival_time=arrival_time,
527 lora_request=lora_request,
517 - )
528 + prompt_adapter_request=prompt_adapter_request)
529 else:
530 raise ValueError(
531 "Either SamplingParams or PoolingParams must be provided.")
@@ -535,6 +546,7 @@ class LLMEngine:
546 request_id: str,
547 inputs: PromptInputs,
548 lora_request: Optional[LoRARequest] = None,
549 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
550 ) -> LLMInputs:
551 if isinstance(inputs, str):
552 inputs = {"prompt": inputs}
@@ -549,6 +561,11 @@ class LLMEngine:
561 else:
562 prompt_token_ids = inputs["prompt_token_ids"]
563
564 + if prompt_adapter_request:
565 + prompt_token_ids = \
566 + [0] * prompt_adapter_request.prompt_adapter_num_virtual_tokens\
567 + + prompt_token_ids
568 +
569 llm_inputs = LLMInputs(prompt_token_ids=prompt_token_ids,
570 prompt=inputs.get("prompt"),
571 multi_modal_data=inputs.get("multi_modal_data"))
@@ -563,6 +580,7 @@ class LLMEngine:
580 arrival_time: Optional[float] = None,
581 lora_request: Optional[LoRARequest] = None,
582 trace_headers: Optional[Dict[str, str]] = None,
583 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
584 ) -> None:
585 """Add a request to the engine's request pool.
586
@@ -612,9 +630,11 @@ class LLMEngine:
630 if arrival_time is None:
631 arrival_time = time.time()
632
615 - processed_inputs = self.process_model_inputs(request_id=request_id,
616 - inputs=inputs,
617 - lora_request=lora_request)
633 + processed_inputs = self.process_model_inputs(
634 + request_id=request_id,
635 + inputs=inputs,
636 + lora_request=lora_request,
637 + prompt_adapter_request=prompt_adapter_request)
638
639 self._add_processed_request(
640 request_id=request_id,
@@ -622,6 +642,7 @@ class LLMEngine:
642 params=params,
643 arrival_time=arrival_time,
644 lora_request=lora_request,
645 + prompt_adapter_request=prompt_adapter_request,
646 trace_headers=trace_headers,
647 )
648
@@ -633,6 +654,7 @@ class LLMEngine:
654 arrival_time: float,
655 lora_request: Optional[LoRARequest],
656 trace_headers: Optional[Dict[str, str]] = None,
657 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
658 ) -> SequenceGroup:
659 """Creates a SequenceGroup with SamplingParams."""
660 max_logprobs = self.get_model_config().max_logprobs
@@ -658,7 +680,7 @@ class LLMEngine:
680 sampling_params=sampling_params,
681 lora_request=lora_request,
682 trace_headers=trace_headers,
661 - )
683 + prompt_adapter_request=prompt_adapter_request)
684
685 return seq_group
686
@@ -669,16 +691,19 @@ class LLMEngine:
691 pooling_params: PoolingParams,
692 arrival_time: float,
693 lora_request: Optional[LoRARequest],
694 + prompt_adapter_request: Optional[PromptAdapterRequest],
695 ) -> SequenceGroup:
696 """Creates a SequenceGroup with PoolingParams."""
697 # Defensive copy of PoolingParams, which are used by the pooler
698 pooling_params = pooling_params.clone()
699 # Create the sequence group.
677 - seq_group = SequenceGroup(request_id=request_id,
678 - seqs=[seq],
679 - arrival_time=arrival_time,
680 - lora_request=lora_request,
681 - pooling_params=pooling_params)
700 + seq_group = SequenceGroup(
701 + request_id=request_id,
702 + seqs=[seq],
703 + arrival_time=arrival_time,
704 + lora_request=lora_request,
705 + pooling_params=pooling_params,
706 + prompt_adapter_request=prompt_adapter_request)
707 return seq_group
708
709 def abort_request(self, request_id: Union[str, Iterable[str]]) -> None:
@@ -1082,6 +1107,16 @@ class LLMEngine:
1107 def pin_lora(self, lora_id: int) -> bool:
1108 return self.model_executor.pin_lora(lora_id)
1109
1110 + def add_prompt_adapter(
1111 + self, prompt_adapter_request: PromptAdapterRequest) -> bool:
1112 + return self.model_executor.add_prompt_adapter(prompt_adapter_request)
1113 +
1114 + def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
1115 + return self.model_executor.remove_prompt_adapter(prompt_adapter_id)
1116 +
1117 + def list_prompt_adapters(self) -> List[int]:
1118 + return self.model_executor.list_prompt_adapters()
1119 +
1120 def check_health(self) -> None:
1121 if self.tokenizer:
1122 self.tokenizer.check_health()
vllm/entrypoints/llm.py
+23 -10
@@ -13,6 +13,7 @@ from vllm.logger import init_logger
13 from vllm.lora.request import LoRARequest
14 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
15 from vllm.pooling_params import PoolingParams
16 +from vllm.prompt_adapter.request import PromptAdapterRequest
17 from vllm.sampling_params import SamplingParams
18 from vllm.transformers_utils.tokenizer import get_cached_tokenizer
19 from vllm.usage.usage_lib import UsageContext
@@ -255,6 +256,7 @@ class LLM:
256 prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
257 use_tqdm: bool = True,
258 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
259 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
260 ) -> List[RequestOutput]:
261 """Generates the completions for the input prompts.
262
@@ -271,6 +273,8 @@ class LLM:
273 prompts and it is paired one by one with the prompt.
274 use_tqdm: Whether to use tqdm to display the progress bar.
275 lora_request: LoRA request to use for generation, if any.
276 + prompt_adapter_request: Prompt Adapter request to use for
277 + generation, if any.
278
279 Returns:
280 A list of `RequestOutput` objects containing the
@@ -304,7 +308,7 @@ class LLM:
308 inputs=inputs,
309 params=sampling_params,
310 lora_request=lora_request,
307 - )
311 + prompt_adapter_request=prompt_adapter_request)
312
313 outputs = self._run_engine(use_tqdm=use_tqdm)
314 return LLMEngine.validate_outputs(outputs, RequestOutput)
@@ -397,6 +401,7 @@ class LLM:
401 prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
402 use_tqdm: bool = True,
403 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
404 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
405 ) -> List[EmbeddingRequestOutput]:
406 """Generates the completions for the input prompts.
407
@@ -412,6 +417,8 @@ class LLM:
417 use the default pooling parameters.
418 use_tqdm: Whether to use tqdm to display the progress bar.
419 lora_request: LoRA request to use for generation, if any.
420 + prompt_adapter_request: Prompt Adapter request to use for
421 + generation, if any.
422
423 Returns:
424 A list of `EmbeddingRequestOutput` objects containing the
@@ -445,6 +452,7 @@ class LLM:
452 inputs=inputs,
453 params=pooling_params,
454 lora_request=lora_request,
455 + prompt_adapter_request=prompt_adapter_request,
456 )
457
458 outputs = self._run_engine(use_tqdm=use_tqdm)
@@ -504,6 +512,7 @@ class LLM:
512 params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
513 Sequence[PoolingParams]],
514 lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
515 + prompt_adapter_request: Optional[PromptAdapterRequest],
516 ) -> None:
517 if isinstance(inputs, (str, dict)):
518 # Convert a single prompt to a list.
@@ -526,19 +535,23 @@ class LLM:
535 params[i] if isinstance(params, Sequence) else params,
536 lora_request=lora_request[i] if isinstance(
537 lora_request, Sequence) else lora_request,
529 - )
538 + prompt_adapter_request=prompt_adapter_request)
539
540 def _add_request(
532 - self,
533 - inputs: PromptInputs,
534 - params: Union[SamplingParams, PoolingParams],
535 - lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
541 + self,
542 + inputs: PromptInputs,
543 + params: Union[SamplingParams, PoolingParams],
544 + lora_request: Optional[Union[List[LoRARequest],
545 + LoRARequest]] = None,
546 + prompt_adapter_request: Optional[PromptAdapterRequest] = None
547 ) -> None:
548 request_id = str(next(self.request_counter))
538 - self.llm_engine.add_request(request_id,
539 - inputs,
540 - params,
541 - lora_request=lora_request)
549 + self.llm_engine.add_request(
550 + request_id,
551 + inputs,
552 + params,
553 + lora_request=lora_request,
554 + prompt_adapter_request=prompt_adapter_request)
555
556 def _run_engine(
557 self, *, use_tqdm: bool
vllm/entrypoints/openai/api_server.py
+3 -2
@@ -116,7 +116,7 @@ async def detokenize(request: DetokenizeRequest):
116
117 @app.get("/v1/models")
118 async def show_available_models():
119 - models = await openai_serving_chat.show_available_models()
119 + models = await openai_serving_completion.show_available_models()
120 return JSONResponse(content=models.model_dump())
121
122
@@ -236,7 +236,8 @@ if __name__ == "__main__":
236 args.lora_modules,
237 args.chat_template)
238 openai_serving_completion = OpenAIServingCompletion(
239 - engine, model_config, served_model_names, args.lora_modules)
239 + engine, model_config, served_model_names, args.lora_modules,
240 + args.prompt_adapters)
241 openai_serving_embedding = OpenAIServingEmbedding(engine, model_config,
242 served_model_names)
243 app.root_path = args.root_path
vllm/entrypoints/openai/cli_args.py
+20 -1
@@ -9,7 +9,8 @@ import json
9 import ssl
10
11 from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
12 -from vllm.entrypoints.openai.serving_engine import LoRAModulePath
12 +from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
13 + PromptAdapterPath)
14 from vllm.utils import FlexibleArgumentParser
15
16
@@ -23,6 +24,16 @@ class LoRAParserAction(argparse.Action):
24 setattr(namespace, self.dest, lora_list)
25
26
27 +class PromptAdapterParserAction(argparse.Action):
28 +
29 + def __call__(self, parser, namespace, values, option_string=None):
30 + adapter_list = []
31 + for item in values:
32 + name, path = item.split('=')
33 + adapter_list.append(PromptAdapterPath(name, path))
34 + setattr(namespace, self.dest, adapter_list)
35 +
36 +
37 def make_arg_parser():
38 parser = FlexibleArgumentParser(
39 description="vLLM OpenAI-Compatible RESTful API server.")
@@ -65,6 +76,14 @@ def make_arg_parser():
76 action=LoRAParserAction,
77 help="LoRA module configurations in the format name=path. "
78 "Multiple modules can be specified.")
79 + parser.add_argument(
80 + "--prompt-adapters",
81 + type=nullable_str,
82 + default=None,
83 + nargs='+',
84 + action=PromptAdapterParserAction,
85 + help="Prompt adapter configurations in the format name=path. "
86 + "Multiple adapters can be specified.")
87 parser.add_argument("--chat-template",
88 type=nullable_str,
89 default=None,
vllm/entrypoints/openai/serving_chat.py
+1 -1
@@ -258,7 +258,7 @@ class OpenAIServingChat(OpenAIServing):
258 prompt=prompt,
259 add_special_tokens=request.add_special_tokens)
260 sampling_params = request.to_sampling_params()
261 - lora_request = self._maybe_get_lora(request)
261 + _, lora_request = self._maybe_get_adapter(request)
262 decoding_config = await self.engine.get_decoding_config()
263 guided_decoding_backend = request.guided_decoding_backend \
264 or decoding_config.guided_decoding_backend
vllm/entrypoints/openai/serving_completion.py
+13 -4
@@ -22,7 +22,8 @@ from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
22 TokenizeResponse, UsageInfo)
23 # yapf: enable
24 from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
25 - OpenAIServing)
25 + OpenAIServing,
26 + PromptAdapterPath)
27 from vllm.logger import init_logger
28 from vllm.model_executor.guided_decoding import (
29 get_guided_decoding_logits_processor)
@@ -67,11 +68,13 @@ class OpenAIServingCompletion(OpenAIServing):
68
69 def __init__(self, engine: AsyncLLMEngine, model_config: ModelConfig,
70 served_model_names: List[str],
70 - lora_modules: Optional[List[LoRAModulePath]]):
71 + lora_modules: Optional[List[LoRAModulePath]],
72 + prompt_adapters: Optional[List[PromptAdapterPath]]):
73 super().__init__(engine=engine,
74 model_config=model_config,
75 served_model_names=served_model_names,
74 - lora_modules=lora_modules)
76 + lora_modules=lora_modules,
77 + prompt_adapters=prompt_adapters)
78
79 async def create_completion(self, request: CompletionRequest,
80 raw_request: Request):
@@ -101,7 +104,12 @@ class OpenAIServingCompletion(OpenAIServing):
104 generators: List[AsyncIterator[RequestOutput]] = []
105 try:
106 sampling_params = request.to_sampling_params()
104 - lora_request = self._maybe_get_lora(request)
107 + adapter_type, adapter_request = self._maybe_get_adapter(request)
108 + lora_request, prompt_adapter_request = None, None
109 + if adapter_type == 'LoRA':
110 + lora_request, prompt_adapter_request = adapter_request, None
111 + elif adapter_type == 'PromptAdapter':
112 + lora_request, prompt_adapter_request = None, adapter_request
113 decoding_config = await self.engine.get_decoding_config()
114 guided_decoding_backend = request.guided_decoding_backend \
115 or decoding_config.guided_decoding_backend
@@ -147,6 +155,7 @@ class OpenAIServingCompletion(OpenAIServing):
155 sampling_params,
156 f"{request_id}-{i}",
157 lora_request=lora_request,
158 + prompt_adapter_request=prompt_adapter_request,
159 trace_headers=trace_headers,
160 )
161
vllm/entrypoints/openai/serving_engine.py
+51 -10
@@ -16,12 +16,19 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
16 ModelPermission, TokenizeRequest)
17 from vllm.logger import init_logger
18 from vllm.lora.request import LoRARequest
19 +from vllm.prompt_adapter.request import PromptAdapterRequest
20 from vllm.sequence import Logprob
21 from vllm.transformers_utils.tokenizer import get_tokenizer
22
23 logger = init_logger(__name__)
24
25
26 +@dataclass
27 +class PromptAdapterPath:
28 + name: str
29 + local_path: str
30 +
31 +
32 @dataclass
33 class LoRAModulePath:
34 name: str
@@ -30,9 +37,14 @@ class LoRAModulePath:
37
38 class OpenAIServing:
39
33 - def __init__(self, engine: AsyncLLMEngine, model_config: ModelConfig,
34 - served_model_names: List[str],
35 - lora_modules: Optional[List[LoRAModulePath]]):
40 + def __init__(
41 + self,
42 + engine: AsyncLLMEngine,
43 + model_config: ModelConfig,
44 + served_model_names: List[str],
45 + lora_modules: Optional[List[LoRAModulePath]],
46 + prompt_adapters: Optional[List[PromptAdapterPath]] = None,
47 + ):
48 super().__init__()
49
50 self.engine = engine
@@ -49,9 +61,8 @@ class OpenAIServing:
61
62 self.served_model_names = served_model_names
63
52 - if lora_modules is None:
53 - self.lora_requests = []
54 - else:
64 + self.lora_requests = []
65 + if lora_modules is not None:
66 self.lora_requests = [
67 LoRARequest(
68 lora_name=lora.name,
@@ -60,6 +71,20 @@ class OpenAIServing:
71 ) for i, lora in enumerate(lora_modules, start=1)
72 ]
73
74 + self.prompt_adapter_requests = []
75 + if prompt_adapters is not None:
76 + for i, prompt_adapter in enumerate(prompt_adapters, start=1):
77 + with open(f"./{prompt_adapter.local_path}"
78 + f"/adapter_config.json") as f:
79 + adapter_config = json.load(f)
80 + num_virtual_tokens = adapter_config["num_virtual_tokens"]
81 + self.prompt_adapter_requests.append(
82 + PromptAdapterRequest(
83 + prompt_adapter_name=prompt_adapter.name,
84 + prompt_adapter_id=i,
85 + prompt_adapter_local_path=prompt_adapter.local_path,
86 + prompt_adapter_num_virtual_tokens=num_virtual_tokens))
87 +
88 async def show_available_models(self) -> ModelList:
89 """Show available models. Right now we only have one model."""
90 model_cards = [
@@ -75,7 +100,14 @@ class OpenAIServing:
100 permission=[ModelPermission()])
101 for lora in self.lora_requests
102 ]
103 + prompt_adapter_cards = [
104 + ModelCard(id=prompt_adapter.prompt_adapter_name,
105 + root=self.served_model_names[0],
106 + permission=[ModelPermission()])
107 + for prompt_adapter in self.prompt_adapter_requests
108 + ]
109 model_cards.extend(lora_cards)
110 + model_cards.extend(prompt_adapter_cards)
111 return ModelList(data=model_cards)
112
113 def create_error_response(
@@ -109,20 +141,29 @@ class OpenAIServing:
141 return None
142 if request.model in [lora.lora_name for lora in self.lora_requests]:
143 return None
144 + if request.model in [
145 + prompt_adapter.prompt_adapter_name
146 + for prompt_adapter in self.prompt_adapter_requests
147 + ]:
148 + return None
149 return self.create_error_response(
150 message=f"The model `{request.model}` does not exist.",
151 err_type="NotFoundError",
152 status_code=HTTPStatus.NOT_FOUND)
153
117 - def _maybe_get_lora(
154 + def _maybe_get_adapter(
155 self, request: Union[CompletionRequest, ChatCompletionRequest,
156 EmbeddingRequest]
120 - ) -> Optional[LoRARequest]:
157 + ) -> Tuple[Optional[str], Optional[Union[LoRARequest,
158 + PromptAdapterRequest]]]:
159 if request.model in self.served_model_names:
122 - return None
160 + return None, None
161 for lora in self.lora_requests:
162 if request.model == lora.lora_name:
125 - return lora
163 + return 'LoRA', lora
164 + for prompt_adapter in self.prompt_adapter_requests:
165 + if request.model == prompt_adapter.prompt_adapter_name:
166 + return 'PromptAdapter', prompt_adapter
167 # if _check_model has been called earlier, this will be unreachable
168 raise ValueError(f"The model `{request.model}` does not exist.")
169
vllm/executor/cpu_executor.py
+15
@@ -7,6 +7,7 @@ from vllm.config import CacheConfig, ModelConfig, SchedulerConfig
7 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
8 from vllm.logger import init_logger
9 from vllm.lora.request import LoRARequest
10 +from vllm.prompt_adapter.request import PromptAdapterRequest
11 from vllm.sequence import ExecuteModelRequest, SamplerOutput
12 from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
13 make_async)
@@ -48,6 +49,7 @@ class CPUExecutor(ExecutorBase):
49 lora_config=self.lora_config,
50 multimodal_config=self.multimodal_config,
51 kv_cache_dtype=self.cache_config.cache_dtype,
52 + prompt_adapter_config=self.prompt_adapter_config,
53 is_driver_worker=True,
54 )
55 self.driver_worker.init_device()
@@ -90,6 +92,19 @@ class CPUExecutor(ExecutorBase):
92 def list_loras(self) -> Set[int]:
93 return self.driver_worker.list_loras()
94
95 + def add_prompt_adapter(
96 + self, prompt_adapter_request: PromptAdapterRequest) -> bool:
97 + return self.driver_worker.add_prompt_adapter(prompt_adapter_request)
98 +
99 + def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
100 + return self.driver_worker.remove_prompt_adapter(prompt_adapter_id)
101 +
102 + def list_prompt_adapters(self) -> Set[int]:
103 + return self.driver_worker.list_prompt_adapters()
104 +
105 + def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
106 + return self.driver_worker.pin_prompt_adapter(prompt_adapter_id)
107 +
108 def check_health(self) -> None:
109 # CPUExecutor will always be healthy as long as
110 # it's running.
vllm/executor/executor_base.py
+25 -2
@@ -4,8 +4,10 @@ from typing import List, Optional, Set, Tuple
4
5 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6 ModelConfig, MultiModalConfig, ParallelConfig,
7 - SchedulerConfig, SpeculativeConfig)
7 + PromptAdapterConfig, SchedulerConfig,
8 + SpeculativeConfig)
9 from vllm.lora.request import LoRARequest
10 +from vllm.prompt_adapter.request import PromptAdapterRequest
11 from vllm.sequence import ExecuteModelRequest, SamplerOutput
12
13
@@ -28,6 +30,7 @@ class ExecutorBase(ABC):
30 lora_config: Optional[LoRAConfig],
31 multimodal_config: Optional[MultiModalConfig],
32 speculative_config: Optional[SpeculativeConfig],
33 + prompt_adapter_config: Optional[PromptAdapterConfig],
34 ) -> None:
35 self.model_config = model_config
36 self.cache_config = cache_config
@@ -38,6 +41,7 @@ class ExecutorBase(ABC):
41 self.device_config = device_config
42 self.multimodal_config = multimodal_config
43 self.speculative_config = speculative_config
44 + self.prompt_adapter_config = prompt_adapter_config
45
46 self._init_executor()
47
@@ -95,6 +99,23 @@ class ExecutorBase(ABC):
99 def list_loras(self) -> Set[int]:
100 raise NotImplementedError
101
102 + @abstractmethod
103 + def add_prompt_adapter(
104 + self, prompt_adapter_request: PromptAdapterRequest) -> bool:
105 + raise NotImplementedError
106 +
107 + @abstractmethod
108 + def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
109 + raise NotImplementedError
110 +
111 + @abstractmethod
112 + def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
113 + raise NotImplementedError # type: ignore
114 +
115 + @abstractmethod
116 + def list_prompt_adapters(self) -> Set[int]:
117 + raise NotImplementedError
118 +
119 @abstractmethod
120 def check_health(self) -> None:
121 """Checks if the executor is healthy. If not, it should raise an
@@ -122,12 +143,14 @@ class ExecutorAsyncBase(ExecutorBase):
143 lora_config: Optional[LoRAConfig],
144 multimodal_config: Optional[MultiModalConfig],
145 speculative_config: Optional[SpeculativeConfig],
146 + prompt_adapter_config: Optional[PromptAdapterConfig],
147 ) -> None:
148 self.pp_locks: Optional[List[asyncio.Lock]] = None
149
150 super().__init__(model_config, cache_config, parallel_config,
151 scheduler_config, device_config, load_config,
130 - lora_config, multimodal_config, speculative_config)
152 + lora_config, multimodal_config, speculative_config,
153 + prompt_adapter_config)
154
155 @abstractmethod
156 async def execute_model_async(
vllm/executor/gpu_executor.py
+21
@@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional, Set, Tuple, Union
3 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4 from vllm.logger import init_logger
5 from vllm.lora.request import LoRARequest
6 +from vllm.prompt_adapter.request import PromptAdapterRequest
7 from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
8 from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
9 make_async)
@@ -45,6 +46,7 @@ class GPUExecutor(ExecutorBase):
46 lora_config=self.lora_config,
47 multimodal_config=self.multimodal_config,
48 speculative_config=self.speculative_config,
49 + prompt_adapter_config=self.prompt_adapter_config,
50 is_driver_worker=(not self.parallel_config)
51 or (rank % self.parallel_config.tensor_parallel_size == 0),
52 )
@@ -107,6 +109,25 @@ class GPUExecutor(ExecutorBase):
109 def list_loras(self) -> Set[int]:
110 return self.driver_worker.list_loras()
111
112 + def add_prompt_adapter(
113 + self, prompt_adapter_request: PromptAdapterRequest) -> bool:
114 + assert prompt_adapter_request.prompt_adapter_id > 0, \
115 + "prompt_adapter_id must be greater than 0."
116 + return self.driver_worker.add_prompt_adapter(prompt_adapter_request)
117 +
118 + def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
119 + assert prompt_adapter_id > 0, \
120 + "prompt_adapter_id must be greater than 0."
121 + return self.driver_worker.remove_prompt_adapter(prompt_adapter_id)
122 +
123 + def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
124 + assert prompt_adapter_id > 0, \
125 + "prompt_adapter_id must be greater than 0."
126 + return self.driver_worker.pin_prompt_adapter(prompt_adapter_id)
127 +
128 + def list_prompt_adapters(self) -> Set[int]:
129 + return self.driver_worker.list_prompt_adapters()
130 +
131 def check_health(self) -> None:
132 # GPUExecutor will always be healthy as long as
133 # it's running.
vllm/executor/ray_xpu_executor.py
+4 -1
@@ -8,7 +8,8 @@ from typing import (TYPE_CHECKING, Any, Awaitable, Dict, List, Optional, Set,
8
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 ModelConfig, MultiModalConfig, ParallelConfig,
11 - SchedulerConfig, SpeculativeConfig)
11 + PromptAdapterConfig, SchedulerConfig,
12 + SpeculativeConfig)
13 from vllm.executor.distributed_gpu_executor import ( # yapf: disable
14 DistributedGPUExecutor, DistributedGPUExecutorAsync)
15 from vllm.executor.ray_utils import RayWorkerWrapper, ray
@@ -44,6 +45,7 @@ class RayXPUExecutor(DistributedGPUExecutor):
45 load_config: LoadConfig,
46 lora_config: Optional[LoRAConfig],
47 multimodal_config: Optional[MultiModalConfig],
48 + prompt_adapter_config: Optional[PromptAdapterConfig],
49 speculative_config: Optional[SpeculativeConfig],
50 ) -> None:
51 assert device_config.device_type == "xpu"
@@ -58,6 +60,7 @@ class RayXPUExecutor(DistributedGPUExecutor):
60 self.scheduler_config = scheduler_config
61 self.device_config = device_config
62 self.multimodal_config = multimodal_config
63 + self.prompt_adapter_config = prompt_adapter_config
64
65 placement_group = self.parallel_config.placement_group
66
vllm/executor/xpu_executor.py
+4 -1
@@ -4,7 +4,8 @@ import torch
4
5 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6 ModelConfig, MultiModalConfig, ParallelConfig,
7 - SchedulerConfig, SpeculativeConfig)
7 + PromptAdapterConfig, SchedulerConfig,
8 + SpeculativeConfig)
9 from vllm.executor.executor_base import ExecutorAsyncBase
10 from vllm.executor.gpu_executor import GPUExecutor
11 from vllm.logger import init_logger
@@ -27,6 +28,7 @@ class XPUExecutor(GPUExecutor):
28 load_config: LoadConfig,
29 lora_config: Optional[LoRAConfig],
30 multimodal_config: Optional[MultiModalConfig],
31 + prompt_adapter_config: Optional[PromptAdapterConfig],
32 speculative_config: Optional[SpeculativeConfig],
33 ) -> None:
34 assert device_config.device_type == "xpu"
@@ -43,6 +45,7 @@ class XPUExecutor(GPUExecutor):
45 self.scheduler_config = scheduler_config
46 self.device_config = device_config
47 self.multimodal_config = multimodal_config
48 + self.prompt_adapter_config = prompt_adapter_config
49 self.speculative_config = None
50
51 # Instantiate the worker and load the model to GPU.
vllm/lora/layers.py
+3 -9
@@ -8,6 +8,7 @@ import torch.nn as nn
8 import torch.nn.functional as F
9 from transformers import PretrainedConfig
10
11 +from vllm.adapter_commons.layers import AdapterMapping
12 from vllm.config import LoRAConfig
13 from vllm.distributed import (get_tensor_model_parallel_rank,
14 get_tensor_model_parallel_world_size,
@@ -134,15 +135,8 @@ def _apply_lora_packed_nslice(
135
136
137 @dataclass
137 -class LoRAMapping:
138 - # Per every token in input_ids:
139 - index_mapping: Tuple[int, ...]
140 - # Per sampled token:
141 - prompt_mapping: Tuple[int, ...]
142 -
143 - def __post_init__(self):
144 - self.index_mapping = tuple(self.index_mapping)
145 - self.prompt_mapping = tuple(self.prompt_mapping)
138 +class LoRAMapping(AdapterMapping):
139 + pass
140
141
142 class BaseLayerWithLoRA(nn.Module):
vllm/lora/models.py
+80 -95
@@ -4,12 +4,17 @@ import math
4 import os
5 import re
6 from dataclasses import dataclass, field
7 -from typing import Callable, Dict, List, Optional, Tuple, Type, Union
7 +from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
8
9 import safetensors.torch
10 import torch
11 from torch import nn
12
13 +from vllm.adapter_commons.models import (AdapterLRUCache, AdapterModel,
14 + AdapterModelManager)
15 +from vllm.adapter_commons.utils import (add_adapter, deactivate_adapter,
16 + get_adapter, list_adapters,
17 + remove_adapter, set_adapter_mapping)
18 from vllm.config import LoRAConfig
19 from vllm.logger import init_logger
20 from vllm.lora.layers import (BaseLayerWithLoRA,
@@ -19,7 +24,7 @@ from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
24 from vllm.lora.utils import (from_layer, from_layer_logits_processor,
25 parse_fine_tuned_lora_name, replace_submodule)
26 from vllm.model_executor.models.interfaces import SupportsLoRA
22 -from vllm.utils import LRUCache, is_pin_memory_available
27 +from vllm.utils import is_pin_memory_available
28
29 logger = init_logger(__name__)
30
@@ -153,7 +158,7 @@ def get_lora_id():
158 return _GLOBAL_LORA_ID
159
160
156 -class LoRAModel:
161 +class LoRAModel(AdapterModel):
162 """A LoRA fine-tuned model."""
163
164 def __init__(
@@ -388,7 +393,7 @@ class LoRAModel:
393 )
394
395
391 -class LoRAModelManager:
396 +class LoRAModelManager(AdapterModelManager):
397 """A manager that manages multiple LoRA-fine-tuned models."""
398
399 def __init__(
@@ -440,8 +445,7 @@ class LoRAModelManager:
445 # base_indices, sampler_indices, sampler_indices_padded,
446 # embeddings_indices
447 self.indices_len: List[Optional[int]] = [None] * 4
443 -
444 - self.model = model
448 + super().__init__(model)
449 if hasattr(self.model, "supported_lora_modules"):
450 self.supported_lora_modules = copy.deepcopy(
451 self.model.supported_lora_modules)
@@ -453,11 +457,11 @@ class LoRAModelManager:
457 self.model.packed_modules_mapping)
458 self.packed_modules: Dict[str, List[str]] = {}
459 self.modules: Dict[str, "BaseLayerWithLoRA"] = {}
456 - self._registered_loras: Dict[int, LoRAModel] = {}
460 # Dict instead of a Set for compatibility with LRUCache.
458 - self._active_loras: Dict[int, None] = {}
461 self._last_mapping: Optional[LoRAMapping] = None
462 self._create_lora_modules()
463 + self.model.lora_manager = self
464 + self.adapter_type = 'LoRa'
465
466 @property
467 def capacity(self) -> int:
@@ -467,15 +471,16 @@ class LoRAModelManager:
471 def lora_slots(self) -> int:
472 return self.lora_config.max_loras
473
470 - def __len__(self) -> int:
471 - return len(self._registered_loras)
474 + @property
475 + def adapter_slots(self) -> int:
476 + return self.lora_slots
477
473 - def activate_lora(
478 + def activate_adapter(
479 self,
480 lora_id: int,
481 ) -> bool:
482 """Move LoRA into a GPU buffer to be used in the forward pass."""
478 - if lora_id in self._active_loras:
483 + if lora_id in self._active_adapters:
484 return False
485 first_free_slot = next(
486 ((i, lora_id) for i, lora_id in enumerate(self.lora_index_to_id)
@@ -483,8 +488,8 @@ class LoRAModelManager:
488 if first_free_slot is None:
489 raise ValueError("No free lora slots")
490 index, _ = first_free_slot
486 - self._active_loras[lora_id] = None
487 - lora_model = self._registered_loras[lora_id]
491 + self._active_adapters[lora_id] = None
492 + lora_model = self._registered_adapters[lora_id]
493 logger.debug("Activating LoRA. int id: %d, slot index: %d",
494 lora_model.id, index)
495 self.lora_index_to_id[index] = lora_model.id
@@ -498,21 +503,13 @@ class LoRAModelManager:
503 module.reset_lora(index)
504 return True
505
501 - def _deactivate_lora(self, lora_id: int):
506 + def _deactivate_adapter(self, lora_id: int):
507 try:
508 index = self.lora_index_to_id.index(lora_id)
509 self.lora_index_to_id[index] = None
510 except ValueError:
511 pass
512
508 - def deactivate_lora(self, lora_id: int) -> bool:
509 - """Remove a LoRA from a GPU buffer."""
510 - if lora_id in self._active_loras:
511 - self._deactivate_lora(lora_id)
512 - self._active_loras.pop(lora_id)
513 - return True
514 - return False
515 -
513 def _set_long_lora_context(self, lora: LoRAModel):
514 if self.long_lora_context is None:
515 return
@@ -528,40 +525,19 @@ class LoRAModelManager:
525 if offsets:
526 self.long_lora_context.offsets_by_lora_id[lora.id] = offsets
527
531 - def _add_lora(self, lora: LoRAModel):
528 + def _add_adapter(self, lora: LoRAModel):
529 self._create_merged_loras_inplace(lora)
533 - self._registered_loras[lora.id] = lora
530 + self._registered_adapters[lora.id] = lora
531 self._set_long_lora_context(lora)
532
536 - def add_lora(self, lora: LoRAModel) -> bool:
537 - """Add a LoRAModel to the manager CPU cache."""
538 - logger.debug(
539 - "Adding lora. Model id: %d, "
540 - "int id: %d, "
541 - "scaling factor: %s", lora.id, lora.id, lora.scaling_factor)
542 - if lora.id not in self._registered_loras:
543 - if len(self._registered_loras) >= self.capacity:
544 - raise RuntimeError("No free LoRA slots.")
545 - self._add_lora(lora)
546 - return True
547 - return False
548 -
549 - def remove_lora(self, lora_id: int) -> bool:
550 - """Remove a LoRAModel from the manager CPU cache."""
551 - # TODO: should we check active lora?
552 - self.deactivate_lora(lora_id)
553 - if self.long_lora_context:
554 - self.long_lora_context.offsets_by_lora_id.pop(lora_id, None)
555 - return bool(self._registered_loras.pop(lora_id, None))
556 -
557 - def pin_lora(self, lora_id: int) -> bool:
533 + def pin_adapter(self, lora_id: int) -> bool:
534 """Pin a LoRAModel in the manager cache."""
535 raise NotImplementedError(
536 "Pinning is not supported in LoRAModelManager."
537 "Use LRUCacheLoRAModelManager for pinning") # type: ignore
538
539 # TODO see if this can be vectorized
564 - def _set_lora_mapping(self, mapping: LoRAMapping) -> None:
540 + def _set_adapter_mapping(self, mapping: LoRAMapping) -> None:
541 (base_indices, sampler_indices, sampler_indices_padded,
542 embeddings_indices, long_lora_offsets_tensor,
543 indices_len) = convert_mapping(mapping, self.lora_index_to_id,
@@ -583,23 +559,11 @@ class LoRAModelManager:
559 # Maintain the reference
560 self.indices_len[:] = indices_len
561
586 - def set_lora_mapping(self, lora_mapping: LoRAMapping) -> None:
587 - if self._last_mapping != lora_mapping:
588 - self._set_lora_mapping(lora_mapping)
589 - self._last_mapping = lora_mapping
590 -
591 - def list_loras(self) -> Dict[int, LoRAModel]:
592 - """List all registered LoRAModels."""
593 - return dict(self._registered_loras)
594 -
595 - def get_lora(self, lora_id: int) -> Optional[LoRAModel]:
596 - return self._registered_loras.get(lora_id, None)
597 -
598 - def remove_all_loras(self):
562 + def remove_all_adapters(self):
563 """Remove all LoRAModels from the manager."""
600 - self._registered_loras.clear()
564 + self._registered_adapters.clear()
565 self.lora_index_to_id = [None] * self.lora_slots
602 - self._active_loras.clear()
566 + self._active_adapters.clear()
567
568 def _create_lora_modules(self):
569 for module_name, module in self.model.named_modules(
@@ -743,18 +707,39 @@ class LoRAModelManager:
707 lora_model.loras[module_name] = PackedLoRALayerWeights.pack(
708 replacement_loras)
709
710 + def deactivate_adapter(self, adapter_id: int) -> bool:
711 + return deactivate_adapter(adapter_id, self._active_adapters,
712 + self._deactivate_adapter)
713 +
714 + def add_adapter(self, adapter: LoRAModel) -> bool:
715 + logger.debug(
716 + "Adding lora. Model id: %d, "
717 + "int id: %d, "
718 + "scaling factor: %s", adapter.id, adapter.id,
719 + adapter.scaling_factor)
720 + return add_adapter(adapter, self._registered_adapters, self.capacity,
721 + self._add_adapter)
722
747 -class LoRALRUCache(LRUCache[LoRAModel]):
723 + def set_adapter_mapping(self, mapping: LoRAMapping) -> None:
724 + self._last_mapping = set_adapter_mapping(mapping, self._last_mapping,
725 + self._set_adapter_mapping)
726 +
727 + def remove_adapter(self, adapter_id: int) -> bool:
728 + return remove_adapter(adapter_id, self._registered_adapters,
729 + self.deactivate_adapter)
730 +
731 + def list_adapters(self) -> Dict[int, Any]:
732 + return list_adapters(self._registered_adapters)
733 +
734 + def get_adapter(self, adapter_id: int) -> Optional[Any]:
735 + return get_adapter(adapter_id, self._registered_adapters)
736 +
737 +
738 +class LoRALRUCache(AdapterLRUCache[LoRAModel]):
739
740 def __init__(self, capacity: int, deactivate_lora_fn: Callable[[int],
741 bool]):
751 - super().__init__(capacity)
752 - self.deactivate_lora_fn = deactivate_lora_fn
753 -
754 - def _on_remove(self, key: int, value: LoRAModel):
755 - logger.debug("Removing LoRA. int id: %d", key)
756 - self.deactivate_lora_fn(key)
757 - return super()._on_remove(key, value)
742 + super().__init__(capacity, deactivate_lora_fn)
743
744
745 class LRUCacheLoRAModelManager(LoRAModelManager):
@@ -770,49 +755,49 @@ class LRUCacheLoRAModelManager(LoRAModelManager):
755 ):
756 super().__init__(model, max_num_seqs, max_num_batched_tokens,
757 vocab_size, lora_config)
773 - self._registered_loras: LoRALRUCache = LoRALRUCache(
774 - self.capacity, self.deactivate_lora)
775 - self._active_loras: LoRALRUCache = LoRALRUCache(
776 - self.lora_slots, self._deactivate_lora)
758 + self._registered_adapters: LoRALRUCache = LoRALRUCache(
759 + self.capacity, self.deactivate_adapter)
760 + self._active_adapters: LoRALRUCache = LoRALRUCache(
761 + self.lora_slots, self._deactivate_adapter)
762
778 - def list_loras(self) -> Dict[int, LoRAModel]:
763 + def list_adapters(self) -> Dict[int, LoRAModel]:
764 """List all registered LoRAModels."""
780 - return dict(self._registered_loras.cache)
765 + return dict(self._registered_adapters.cache)
766
782 - def add_lora(self, lora: LoRAModel) -> bool:
767 + def add_adapter(self, lora: LoRAModel) -> bool:
768 """Add a LoRAModel to the manager."""
769 logger.debug(
770 "Adding lora. Model id: %d, "
771 "int id: %d, "
772 "scaling factor: %s", lora.id, lora.id, lora.scaling_factor)
788 - if lora.id not in self._registered_loras:
789 - self._add_lora(lora)
773 + if lora.id not in self._registered_adapters:
774 + self._add_adapter(lora)
775 was_added = True
776 else:
777 # We always touch to update the LRU cache order
793 - self._registered_loras.touch(lora.id)
778 + self._registered_adapters.touch(lora.id)
779 was_added = False
780 return was_added
781
797 - def activate_lora(
782 + def activate_adapter(
783 self,
784 lora_id: int,
785 ) -> bool:
801 - if lora_id not in self._active_loras and len(
802 - self._active_loras) >= self.lora_slots:
803 - self._active_loras.remove_oldest()
804 - result = super().activate_lora(lora_id)
786 + if lora_id not in self._active_adapters and len(
787 + self._active_adapters) >= self.lora_slots:
788 + self._active_adapters.remove_oldest()
789 + result = super().activate_adapter(lora_id)
790 # We always touch to update the LRU cache order
806 - self._active_loras.touch(lora_id)
791 + self._active_adapters.touch(lora_id)
792 return result
793
809 - def remove_oldest_lora(self) -> bool:
810 - if len(self._registered_loras) > 0:
811 - self._registered_loras.remove_oldest()
794 + def remove_oldest_adapter(self) -> bool:
795 + if len(self._registered_adapters) > 0:
796 + self._registered_adapters.remove_oldest()
797 return True
798 return False
799
815 - def pin_lora(self, lora_id: int) -> bool:
800 + def pin_adapter(self, lora_id: int) -> bool:
801 """Pin a LoRAModel in the manager cache."""
802 self._pin_lora_in_cpu_cache(lora_id)
803 self._pin_lora_in_gpu_cache(lora_id)
@@ -820,17 +805,17 @@ class LRUCacheLoRAModelManager(LoRAModelManager):
805
806 def _pin_lora_in_cpu_cache(self, lora_id: int):
807 try:
823 - self._registered_loras.pin(lora_id)
808 + self._registered_adapters.pin(lora_id)
809 except ValueError as err:
810 raise ValueError("Pinning failed. "
811 f"LoRA {lora_id} is not registered.") from err
812
813 def _pin_lora_in_gpu_cache(self, lora_id: int):
829 - if lora_id not in self._active_loras:
814 + if lora_id not in self._active_adapters:
815 # move lora to gpu if not already active
831 - self.activate_lora(lora_id)
816 + self.activate_adapter(lora_id)
817
833 - self._active_loras.pin(lora_id)
818 + self._active_adapters.pin(lora_id)
819
820
821 def create_lora_manager(
vllm/lora/request.py
+14 -11
@@ -1,13 +1,15 @@
1 from dataclasses import dataclass
2 from typing import Optional
3
4 +from vllm.adapter_commons.request import AdapterRequest
5 +
6
7 @dataclass
6 -class LoRARequest:
8 +class LoRARequest(AdapterRequest):
9 """
10 Request for a LoRA adapter.
11
10 - Note that this class should be be used internally. For online
12 + Note that this class should be used internally. For online
13 serving, it is recommended to not allow users to use this class but
14 instead provide another layer of abstraction to prevent users from
15 accessing unauthorized LoRA adapters.
@@ -20,15 +22,16 @@ class LoRARequest:
22 lora_int_id: int
23 lora_local_path: str
24 long_lora_max_len: Optional[int] = None
25 + __hash__ = AdapterRequest.__hash__
26
24 - def __post_init__(self):
25 - if self.lora_int_id < 1:
26 - raise ValueError(
27 - f"lora_int_id must be > 0, got {self.lora_int_id}")
27 + @property
28 + def adapter_id(self):
29 + return self.lora_int_id
30
29 - def __eq__(self, value: object) -> bool:
30 - return isinstance(
31 - value, LoRARequest) and self.lora_int_id == value.lora_int_id
31 + @property
32 + def name(self):
33 + return self.lora_name
34
33 - def __hash__(self) -> int:
34 - return self.lora_int_id
35 + @property
36 + def local_path(self):
37 + return self.lora_local_path
vllm/lora/worker_manager.py
+70 -145
@@ -1,12 +1,15 @@
1 -from abc import ABC, abstractmethod
1 from contextlib import contextmanager
2 from typing import Any, Dict, List, Literal, Optional, Set, Type, Union
3
4 import torch
5
6 +from vllm.adapter_commons.utils import (add_adapter_worker,
7 + apply_adapters_worker,
8 + list_adapters_worker,
9 + set_active_adapters_worker)
10 +from vllm.adapter_commons.worker_manager import AbstractWorkerManager
11 from vllm.config import LoRAConfig
12 from vllm.logger import init_logger
9 -from vllm.lora.layers import LoRAMapping
13 from vllm.lora.models import (LoRAModel, LoRAModelManager,
14 LRUCacheLoRAModelManager, create_lora_manager)
15 from vllm.lora.request import LoRARequest
@@ -14,79 +17,13 @@ from vllm.lora.request import LoRARequest
17 logger = init_logger(__name__)
18
19
17 -class AbstractWorkerLoRAManager(ABC):
18 - """Abstract class for managing LoRA models on the worker side."""
19 -
20 - def __init__(self,
21 - max_num_seqs: int,
22 - max_num_batched_tokens: int,
23 - vocab_size: int,
24 - lora_config: LoRAConfig,
25 - device: torch.device,
26 - max_position_embeddings: Optional[int] = None):
27 - self.max_num_seqs = max_num_seqs
28 - self.max_num_batched_tokens = max_num_batched_tokens
29 - self.max_position_embeddings = max_position_embeddings
30 - self.vocab_size = vocab_size
31 - self.device = device
32 - self.lora_config = lora_config
33 -
34 - # If False, do not cache. If None, cache is empty.
35 - self._cached_dummy_lora: Union[None, Literal[False], LoRAModel] = False
36 -
37 - @contextmanager
38 - def dummy_lora_cache(self):
39 - """Use this context manager to reuse the dummy lora model
40 - to avoid creating it repeatedly."""
41 - self._cached_dummy_lora = None
42 - yield
43 - self._cached_dummy_lora = False
44 -
45 - @property
46 - @abstractmethod
47 - def is_enabled(self) -> bool:
48 - ...
49 -
50 - @abstractmethod
51 - def create_lora_manager(
52 - self,
53 - model: torch.nn.Module,
54 - ) -> Any:
55 - ...
56 -
57 - @abstractmethod
58 - def set_active_loras(self, lora_requests: Set[LoRARequest],
59 - lora_mapping: LoRAMapping) -> None:
60 - ...
61 -
62 - @abstractmethod
63 - def add_lora(self, lora_request: LoRARequest) -> bool:
64 - ...
65 -
66 - @abstractmethod
67 - def add_dummy_lora(self, lora_request: LoRARequest, rank: int) -> bool:
68 - ...
69 -
70 - @abstractmethod
71 - def remove_lora(self, lora_id: int) -> bool:
72 - ...
73 -
74 - @abstractmethod
75 - def remove_all_loras(self):
76 - ...
77 -
78 - @abstractmethod
79 - def list_loras(self) -> Set[int]:
80 - ...
81 -
82 -
83 -class WorkerLoRAManager(AbstractWorkerLoRAManager):
20 +class WorkerLoRAManager(AbstractWorkerManager):
21 """WorkerLoRAManager that manages LoRA models on the worker side.
22
23 Every request, the requested LoRAs will be loaded (unless they are already
24 loaded), and every other LoRA will be unloaded."""
25
89 - _lora_manager_cls: Type[LoRAModelManager] = LoRAModelManager
26 + _manager_cls: Type[LoRAModelManager] = LoRAModelManager
27
28 def __init__(
29 self,
@@ -103,16 +40,23 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
40 self._lora_model_cls = lora_model_cls
41 self.embedding_modules = embedding_modules
42 self.embedding_padding_modules = embedding_padding_modules
43 + self._cached_dummy_lora: Union[None, Literal[False], LoRAModel] = False
44 + self.max_num_seqs = max_num_seqs
45 + self.max_num_batched_tokens = max_num_batched_tokens
46 + self.vocab_size = vocab_size
47 + self.lora_config = lora_config
48 + self.max_position_embeddings = max_position_embeddings
49 + super().__init__(device)
50 # Lazily initialized by create_lora_manager.
107 - self._lora_manager: LoRAModelManager
108 - super().__init__(
109 - max_num_seqs,
110 - max_num_batched_tokens,
111 - vocab_size,
112 - lora_config,
113 - device,
114 - max_position_embeddings=max_position_embeddings,
115 - )
51 + self._adapter_manager: LoRAModelManager
52 +
53 + @contextmanager
54 + def dummy_lora_cache(self):
55 + """Use this context manager to reuse the dummy lora model
56 + to avoid creating it repeatedly."""
57 + self._cached_dummy_lora = None
58 + yield
59 + self._cached_dummy_lora = False
60
61 @property
62 def is_enabled(self) -> bool:
@@ -128,41 +72,14 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
72 max_num_batched_tokens=self.max_num_batched_tokens,
73 vocab_size=self.vocab_size,
74 lora_config=self.lora_config,
131 - lora_manager_cls=self._lora_manager_cls,
75 + lora_manager_cls=self._manager_cls,
76 )
133 - self._lora_manager = lora_manager
77 + self._adapter_manager = lora_manager
78 return lora_manager.model
79
136 - def set_active_loras(self, lora_requests: Set[LoRARequest],
137 - lora_mapping: LoRAMapping) -> None:
138 - self._apply_loras(lora_requests)
139 - self._lora_manager.set_lora_mapping(lora_mapping)
140 -
141 - def _apply_loras(self, lora_requests: Set[LoRARequest]) -> None:
142 - loras_that_exist = self.list_loras()
143 - loras_map = {
144 - lora_request.lora_int_id: lora_request
145 - for lora_request in lora_requests if lora_request
146 - }
147 - if len(loras_map) > self._lora_manager.lora_slots:
148 - raise RuntimeError(
149 - f"Number of requested LoRAs ({len(loras_map)}) is greater "
150 - "than the number of GPU LoRA slots "
151 - f"({self._lora_manager.lora_slots}).")
152 -
153 - new_loras = set(loras_map)
154 - loras_to_add = new_loras - loras_that_exist
155 - loras_to_remove = loras_that_exist - new_loras
156 -
157 - for lora_id in loras_to_remove:
158 - self.remove_lora(lora_id)
159 -
160 - for lora_id in loras_to_add:
161 - self.add_lora(loras_map[lora_id])
162 -
163 - def _load_lora(self, lora_request: LoRARequest) -> LoRAModel:
80 + def _load_adapter(self, lora_request: LoRARequest) -> LoRAModel:
81 try:
165 - model = self._lora_manager.model
82 + model = self._adapter_manager.model
83 supported_lora_modules = model.supported_lora_modules
84 packed_modules_mapping = model.packed_modules_mapping
85 expected_lora_modules: List[str] = []
@@ -198,37 +115,45 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
115 return lora
116
117 def add_dummy_lora(self, lora_request: LoRARequest, rank: int) -> bool:
201 - if lora_request.lora_int_id in self.list_loras():
118 + if lora_request.lora_int_id in self.list_adapters():
119 return False
120 if isinstance(self._cached_dummy_lora, LoRAModel):
121 dummy_lora = self._cached_dummy_lora.clone(
122 lora_request.lora_int_id)
123 else:
207 - dummy_lora = self._lora_manager.create_dummy_lora(
124 + dummy_lora = self._adapter_manager.create_dummy_lora(
125 lora_request.lora_int_id, rank, 1, self.embedding_modules)
126 if self._cached_dummy_lora is None:
127 self._cached_dummy_lora = dummy_lora
211 - return self._lora_manager.add_lora(dummy_lora)
128 + return self._adapter_manager.add_adapter(dummy_lora)
129
213 - def add_lora(self, lora_request: LoRARequest) -> bool:
214 - if lora_request.lora_int_id in self.list_loras():
215 - return False
216 - lora = self._load_lora(lora_request)
217 - loaded = self._lora_manager.add_lora(lora)
218 - self._lora_manager.activate_lora(lora.id)
219 - return loaded
130 + def pin_adapter(self, adapter_id: int) -> bool:
131 + return self._adapter_manager.pin_adapter(adapter_id)
132 +
133 + def set_active_adapters(self, requests: Set[Any],
134 + mapping: Optional[Any]) -> None:
135 + set_active_adapters_worker(requests, mapping, self._apply_adapters,
136 + self._adapter_manager.set_adapter_mapping)
137 +
138 + def _apply_adapters(self, adapter_requests: Set[Any]) -> None:
139 + apply_adapters_worker(adapter_requests, self.list_adapters,
140 + self._adapter_manager.adapter_slots,
141 + self.remove_adapter, self.add_adapter)
142
221 - def remove_lora(self, lora_id: int) -> bool:
222 - return self._lora_manager.remove_lora(lora_id)
143 + def add_adapter(self, adapter_request: Any) -> bool:
144 + return add_adapter_worker(adapter_request, self.list_adapters,
145 + self._load_adapter,
146 + self._adapter_manager.add_adapter,
147 + self._adapter_manager.activate_adapter)
148
224 - def pin_lora(self, lora_id: int) -> bool:
225 - return self._lora_manager.pin_lora(lora_id)
149 + def remove_adapter(self, adapter_id: int) -> bool:
150 + return self._adapter_manager.remove_adapter(adapter_id)
151
227 - def remove_all_loras(self):
228 - self._lora_manager.remove_all_loras()
152 + def remove_all_adapters(self):
153 + self._adapter_manager.remove_all_adapters()
154
230 - def list_loras(self) -> Set[int]:
231 - return set(self._lora_manager.list_loras())
155 + def list_adapters(self) -> Set[int]:
156 + return list_adapters_worker(self._adapter_manager.list_adapters)
157
158
159 class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
@@ -238,8 +163,7 @@ class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
163 (unless they are already loaded) and least recently used LoRAs will
164 be unloaded if the cache is above capacity."""
165
241 - _lora_manager_cls: Type[
242 - LRUCacheLoRAModelManager] = LRUCacheLoRAModelManager
166 + _manager_cls: Type[LRUCacheLoRAModelManager] = LRUCacheLoRAModelManager
167
168 def create_lora_manager(
169 self,
@@ -247,40 +171,41 @@ class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
171 ) -> Any:
172 lora_manager = create_lora_manager(
173 model,
250 - lora_manager_cls=self._lora_manager_cls,
174 + lora_manager_cls=self._manager_cls,
175 max_num_seqs=self.max_num_seqs,
176 vocab_size=self.vocab_size,
177 lora_config=self.lora_config,
178 max_num_batched_tokens=self.max_num_batched_tokens,
179 )
256 - self._lora_manager = lora_manager
180 + self._adapter_manager = lora_manager
181 return lora_manager.model
182
259 - def _apply_loras(self, lora_requests: Set[LoRARequest]) -> None:
183 + def _apply_adapters(self, lora_requests: Set[LoRARequest]) -> None:
184 loras_map = {
185 lora_request.lora_int_id: lora_request
186 for lora_request in lora_requests if lora_request
187 }
264 - if len(loras_map) > self._lora_manager.lora_slots:
188 + if len(loras_map) > self._adapter_manager.lora_slots:
189 raise RuntimeError(
190 f"Number of requested LoRAs ({len(loras_map)}) is greater "
191 "than the number of GPU LoRA slots "
268 - f"({self._lora_manager.lora_slots}).")
192 + f"({self._adapter_manager.lora_slots}).")
193 for lora in loras_map.values():
270 - self.add_lora(lora)
194 + self.add_adapter(lora)
195
272 - def add_lora(self, lora_request: LoRARequest) -> bool:
273 - if lora_request.lora_int_id not in self.list_loras():
196 + def add_adapter(self, lora_request: LoRARequest) -> bool:
197 + if lora_request.lora_int_id not in self.list_adapters():
198 # Remove before we load the new lora to save memory
275 - if len(self._lora_manager) + 1 > self._lora_manager.capacity:
276 - assert isinstance(self._lora_manager, LRUCacheLoRAModelManager)
277 - self._lora_manager.remove_oldest_lora()
278 - lora = self._load_lora(lora_request)
279 - loaded = self._lora_manager.add_lora(lora)
199 + if len(self._adapter_manager) + 1 > self._adapter_manager.capacity:
200 + assert isinstance(self._adapter_manager,
201 + LRUCacheLoRAModelManager)
202 + self._adapter_manager.remove_oldest_adapter()
203 + lora = self._load_adapter(lora_request)
204 + loaded = self._adapter_manager.add_adapter(lora)
205 else:
206 # If the lora is already loaded, just touch it to
207 # update its position in the caches
283 - loaded = self._lora_manager.get_lora(
208 + loaded = self._adapter_manager.get_adapter(
209 lora_request.lora_int_id) is not None
285 - self._lora_manager.activate_lora(lora_request.lora_int_id)
210 + self._adapter_manager.activate_adapter(lora_request.lora_int_id)
211 return loaded
vllm/prompt_adapter/__init__.py
vllm/prompt_adapter/layers.py new
+80
@@ -0,0 +1,80 @@
1 +from dataclasses import dataclass
2 +from typing import Optional
3 +
4 +import torch
5 +from torch import nn
6 +
7 +from vllm.adapter_commons.layers import AdapterMapping
8 +from vllm.config import PromptAdapterConfig
9 +from vllm.model_executor.layers.vocab_parallel_embedding import (
10 + VocabParallelEmbedding)
11 +
12 +
13 +@dataclass
14 +class PromptAdapterMapping(AdapterMapping):
15 + pass
16 +
17 +
18 +class VocabParallelEmbeddingWithPromptAdapter(nn.Module):
19 +
20 + def __init__(self, base_layer: VocabParallelEmbedding) -> None:
21 + super().__init__()
22 + self.base_layer = base_layer
23 + self.emb_layer = self.base_layer
24 + if 'LoRA' in base_layer.__class__.__name__:
25 + self.emb_layer = self.base_layer.base_layer
26 +
27 + def create_prompt_adapter_weights(
28 + self, prompt_adapter_config: PromptAdapterConfig):
29 + self.embeddings_tensors = torch.zeros(
30 + (
31 + prompt_adapter_config.max_prompt_adapters,
32 + prompt_adapter_config.max_prompt_adapter_token,
33 + self.emb_layer.embedding_dim,
34 + ),
35 + dtype=self.emb_layer.weight.dtype,
36 + device=self.emb_layer.weight.device,
37 + )
38 + self.adapter_lengths = torch.zeros(
39 + prompt_adapter_config.max_prompt_adapters,
40 + dtype=torch.long,
41 + device=self.emb_layer.weight.device)
42 +
43 + self.indices_gpu: torch.Tensor
44 + self.embedding_indices_gpu: torch.Tensor
45 +
46 + def reset_prompt_adapter(self, index: int):
47 + self.embeddings_tensors[index] = 0
48 +
49 + def set_prompt_adapter(
50 + self,
51 + index: int,
52 + adapter_model: Optional[torch.Tensor],
53 + ):
54 + self.reset_prompt_adapter(index)
55 + if adapter_model is not None:
56 + length = adapter_model.shape[0]
57 + self.embeddings_tensors[index, :length] = adapter_model
58 + self.adapter_lengths[index] = length
59 +
60 + def set_mapping(
61 + self,
62 + prompt_indices: torch.Tensor,
63 + prompt_embedding_indices: torch.Tensor,
64 + ):
65 + self.indices_gpu = prompt_indices.to(
66 + device=self.emb_layer.weight.device)
67 + self.embedding_indices_gpu = prompt_embedding_indices.to(
68 + device=self.emb_layer.weight.device)
69 +
70 + def forward(self, x: torch.Tensor) -> torch.Tensor:
71 + hidden_states = self.base_layer(x)
72 + if self.embedding_indices_gpu.ndim > 1:
73 + valid_mask = self.indices_gpu != -1
74 + gathered_embeddings = self.embeddings_tensors[
75 + self.embedding_indices_gpu[:, 0],
76 + self.embedding_indices_gpu[:, 1]]
77 +
78 + # Update hidden states
79 + hidden_states[valid_mask] = gathered_embeddings
80 + return hidden_states
\ No newline at end of file
vllm/prompt_adapter/models.py new
+355
@@ -0,0 +1,355 @@
1 +import logging
2 +import math
3 +from typing import Any, Callable, Dict, List, Optional, Type
4 +
5 +import torch
6 +from torch import nn
7 +
8 +from vllm.adapter_commons.models import (AdapterLRUCache, AdapterModel,
9 + AdapterModelManager)
10 +from vllm.adapter_commons.utils import (add_adapter, deactivate_adapter,
11 + get_adapter, list_adapters,
12 + remove_adapter, set_adapter_mapping)
13 +from vllm.config import PromptAdapterConfig
14 +from vllm.prompt_adapter.layers import (
15 + VocabParallelEmbeddingWithPromptAdapter) # yapf: disable
16 +from vllm.prompt_adapter.layers import PromptAdapterMapping
17 +
18 +logger = logging.getLogger(__name__)
19 +
20 +_GLOBAL_PROMPT_ADAPTER_ID = 0
21 +
22 +
23 +def get_prompt_adapter_id():
24 + global _GLOBAL_PROMPT_ADAPTER_ID
25 + _GLOBAL_PROMPT_ADAPTER_ID += 1
26 + return _GLOBAL_PROMPT_ADAPTER_ID
27 +
28 +
29 +def convert_to_embedding_indices(indices):
30 + embedding_indices = []
31 + count = 0
32 +
33 + for value in indices:
34 + if value == -1:
35 + count = 0
36 + else:
37 + embedding_indices.append([value, count])
38 + count += 1
39 +
40 + return torch.tensor(embedding_indices)
41 +
42 +
43 +def convert_mapping(
44 + mapping: PromptAdapterMapping,
45 + prompt_adapter_index_to_id: List[Optional[int]],
46 +) -> torch.Tensor:
47 + """Converts PromptAdapterMapping to index tensors.
48 +
49 + Args:
50 + mapping: PromptAdapterMapping mapping rows in a
51 + batch to PromptAdapter ids.
52 + prompt_adapter_index_to_id: List mapping PromptAdapter
53 + ids to PromptAdapter indices.
54 +
55 + Returns:
56 + pa_indices: Tensor of shape [batch_size] mapping batch rows to
57 + PromptAdapter indices.
58 + """
59 + id_to_index = {
60 + id_: idx
61 + for idx, id_ in enumerate(prompt_adapter_index_to_id)
62 + if id_ is not None
63 + }
64 + pa_indices = ([
65 + id_to_index.get(id_, -1) if id_ > 0 else -1
66 + for id_ in mapping.index_mapping
67 + ])
68 +
69 + pa_embedding_mapping = convert_to_embedding_indices(pa_indices)
70 + pa_indices = torch.tensor(pa_indices)
71 + return pa_indices, pa_embedding_mapping
72 +
73 +
74 +class PromptAdapterModel(AdapterModel):
75 +
76 + def __init__(self,
77 + prompt_adapter_id=None,
78 + num_virtual_tokens=None,
79 + prompt_embedding=None) -> None:
80 + self.id = prompt_adapter_id
81 + self.prompt_embedding = prompt_embedding
82 + self.num_virtual_tokens = num_virtual_tokens
83 +
84 + @classmethod
85 + def from_local_checkpoint(
86 + cls,
87 + adapter_model_path: str,
88 + prompt_adapter_id: int,
89 + num_virtual_tokens: int,
90 + config: PromptAdapterConfig,
91 + device: str = "cuda",
92 + ) -> "PromptAdapterModel":
93 + from peft.utils import load_peft_weights
94 +
95 + if num_virtual_tokens > config.max_prompt_adapter_token:
96 + raise ValueError(
97 + f'num_virtual_tokens ({num_virtual_tokens}) should be <= '
98 + f'max_prompt_adapter_token({config.max_prompt_adapter_token})')
99 +
100 + adapters_weights = load_peft_weights(adapter_model_path, device)
101 + prompt_embedding = adapters_weights["prompt_embeddings"].to(
102 + config.prompt_adapter_dtype)
103 +
104 + return cls(prompt_adapter_id, num_virtual_tokens, prompt_embedding)
105 +
106 +
107 +class PromptAdapterModelManager(AdapterModelManager):
108 + """A manager that manages multiple Prompt Adapter models."""
109 +
110 + def __init__(
111 + self,
112 + model: nn.Module,
113 + max_num_seqs: int,
114 + max_num_batched_tokens: int,
115 + prompt_adapter_config: PromptAdapterConfig,
116 + ):
117 + """Create a PromptAdapterModel and adapter for a given model.
118 +
119 + Args:
120 + model: the model to be adapted.
121 + max_num_seqs: the maximum number of sequences model can run in a
122 + single batch.
123 + max_num_batched_tokens: the maximum number of tokens model can run
124 + in a single batch.
125 + prompt_adapter_config: the PromptAdapter config,
126 + """
127 + self.model: nn.Module = model
128 + # Dict instead of a Set for compatibility with LRUCache.
129 + self.prompt_adapter_index_to_id: List[
130 + Optional[int]] = [None] * self.prompt_adapter_slots
131 + self.max_num_seqs = max_num_seqs
132 + self.max_num_batched_tokens = math.ceil(max_num_batched_tokens / 8) * 8
133 + self.prompt_adapter_config = prompt_adapter_config
134 + self.model.prompt_adapter_manager = self
135 + self.adapter_type = 'PromptAdapter'
136 +
137 + self.base_indices = torch.tensor([-1])
138 + self.base_embedding_indices = torch.tensor([])
139 +
140 + self.modules: Dict[str, nn.Module] = {}
141 + self._create_prompt_adapter_modules()
142 + self._last_mapping: Optional[PromptAdapterMapping] = None
143 +
144 + @property
145 + def prompt_adapter_slots(self) -> int:
146 + return self.prompt_adapter_config.max_prompt_adapters
147 +
148 + @property
149 + def adapter_slots(self) -> int:
150 + return self.prompt_adapter_slots
151 +
152 + @property
153 + def capacity(self) -> int:
154 + return self.prompt_adapter_config.max_cpu_prompt_adapters
155 +
156 + def activate_adapter(
157 + self,
158 + prompt_adapter_id: int,
159 + ) -> bool:
160 + """Move PromptAdapter into a GPU buffer
161 + to be used in the forward pass."""
162 + if prompt_adapter_id in self._active_adapters:
163 + return False
164 + first_free_slot = next(
165 + ((i, prompt_adapter_id) for i, prompt_adapter_id in enumerate(
166 + self.prompt_adapter_index_to_id) if prompt_adapter_id is None),
167 + None)
168 + if first_free_slot is None:
169 + raise ValueError("No free prompt_adapter slots")
170 + index, _ = first_free_slot
171 + self._active_adapters[prompt_adapter_id] = None
172 + prompt_adapter_model = (self._registered_adapters[prompt_adapter_id])
173 + logger.debug("Activating prompt_adapter. int id: %d, slot index: %d",
174 + prompt_adapter_model.id, index)
175 + self.prompt_adapter_index_to_id[index] = prompt_adapter_model.id
176 + for _, v in self.modules.items():
177 + v.set_prompt_adapter(index, prompt_adapter_model.prompt_embedding)
178 + return True
179 +
180 + def _deactivate_adapter(self, prompt_adapter_id: int):
181 + try:
182 + index = self.prompt_adapter_index_to_id.index(prompt_adapter_id)
183 + self.prompt_adapter_index_to_id[index] = None
184 + for _, v in self.modules.items():
185 + v.reset_prompt_adapter(index)
186 + except ValueError:
187 + pass
188 +
189 + def _add_adapter(self, prompt_adapter: PromptAdapterModel):
190 + self._registered_adapters[prompt_adapter.id] = prompt_adapter
191 +
192 + def _set_adapter_mapping(self, mapping: PromptAdapterMapping) -> None:
193 + base_indices, base_embedding_indices = convert_mapping(
194 + mapping, self.prompt_adapter_index_to_id)
195 + for k, v in self.modules.items():
196 + v.set_mapping(base_indices, base_embedding_indices)
197 +
198 + def _create_prompt_adapter_modules(self):
199 + for module_name, module in self.model.named_modules(
200 + remove_duplicate=False):
201 + if "VocabParallel" in module.__class__.__name__:
202 + new_module = VocabParallelEmbeddingWithPromptAdapter(module)
203 + new_module.create_prompt_adapter_weights(
204 + self.prompt_adapter_config)
205 + replaced_module = self.replace_submodule(
206 + self.model, module_name, new_module)
207 + self.register_module(module.__class__.__name__,
208 + replaced_module)
209 + replaced_module.set_mapping(self.base_indices,
210 + self.base_embedding_indices)
211 + break
212 +
213 + def replace_submodule(self, model: nn.Module, module_name: str,
214 + new_module: nn.Module) -> nn.Module:
215 + """Replace a submodule in a model with a new module."""
216 + parent = model.get_submodule(".".join(module_name.split(".")[:-1]))
217 + target_name = module_name.split(".")[-1]
218 + setattr(parent, target_name, new_module)
219 + return new_module
220 +
221 + def register_module(self, module_name: str, module: nn.Module):
222 + self.modules[module_name] = module
223 +
224 + def pin_adapter(self, prompt_adapter_id: int) -> bool:
225 + """Pin a PromptAdapterModel in the manager cache."""
226 + raise NotImplementedError(
227 + "Pinning is not supported in PromptAdapterModelManager."
228 + "Use LRUCachePromptAdapterModelManager for pinning"
229 + ) # type: ignore
230 +
231 + def remove_all_adapters(self):
232 + """Remove all PromptAdapterModel from the manager."""
233 + self._registered_adapters.clear()
234 + self.prompt_adapter_index_to_id = [None] * self.prompt_adapter_slots
235 + self._active_adapters.clear()
236 +
237 + def deactivate_adapter(self, adapter_id: int) -> bool:
238 + return deactivate_adapter(adapter_id, self._active_adapters,
239 + self._deactivate_adapter)
240 +
241 + def add_adapter(self, adapter: PromptAdapterModel) -> bool:
242 + return add_adapter(adapter, self._registered_adapters, self.capacity,
243 + self._add_adapter)
244 +
245 + def set_adapter_mapping(self, mapping: PromptAdapterMapping) -> None:
246 + self._last_mapping = set_adapter_mapping(mapping, self._last_mapping,
247 + self._set_adapter_mapping)
248 +
249 + def remove_adapter(self, adapter_id: int) -> bool:
250 + return remove_adapter(adapter_id, self._registered_adapters,
251 + self.deactivate_adapter)
252 +
253 + def list_adapters(self) -> Dict[int, Any]:
254 + return list_adapters(self._registered_adapters)
255 +
256 + def get_adapter(self, adapter_id: int) -> Optional[Any]:
257 + return get_adapter(adapter_id, self._registered_adapters)
258 +
259 +
260 +class PromptAdapterLRUCache(AdapterLRUCache[PromptAdapterModel]):
261 +
262 + def __init__(self, capacity: int,
263 + deactivate_prompt_adapter_fn: Callable[[int], bool]):
264 + super().__init__(capacity, deactivate_prompt_adapter_fn)
265 +
266 +
267 +class LRUCachePromptAdapterModelManager(PromptAdapterModelManager):
268 + """A model manager that manages multiple prompt_adapters with LRU cache."""
269 +
270 + def __init__(
271 + self,
272 + model: nn.Module,
273 + max_num_seqs: int,
274 + max_num_batched_tokens: int,
275 + prompt_adapter_config: PromptAdapterConfig,
276 + ):
277 + self.prompt_adapter_config = prompt_adapter_config
278 + super().__init__(model, max_num_seqs, max_num_batched_tokens,
279 + prompt_adapter_config)
280 + self._registered_adapters = PromptAdapterLRUCache(
281 + self.capacity, self.deactivate_adapter)
282 + self._active_adapters = PromptAdapterLRUCache(
283 + self.prompt_adapter_slots, self._deactivate_adapter)
284 +
285 + def list_adapters(self) -> Dict[int, PromptAdapterModel]:
286 + """List all registered PromptAdapterModel."""
287 + return dict(self._registered_adapters.cache)
288 +
289 + def add_adapter(self, prompt_adapter: PromptAdapterModel) -> bool:
290 + """Add a PromptAdapterModel to the manager."""
291 + if prompt_adapter.id not in self._registered_adapters:
292 + self._add_adapter(prompt_adapter)
293 + was_added = True
294 + else:
295 + # We always touch to update the LRU cache order
296 + self._registered_adapters.touch(prompt_adapter.id)
297 + was_added = False
298 + return was_added
299 +
300 + def activate_adapter(
301 + self,
302 + prompt_adapter_id: int,
303 + ) -> bool:
304 + if prompt_adapter_id not in self._active_adapters and len(
305 + self._active_adapters) >= self.prompt_adapter_slots:
306 + self._active_adapters.remove_oldest()
307 + result = super().activate_adapter(prompt_adapter_id)
308 + # We always touch to update the LRU cache order
309 + self._active_adapters.touch(prompt_adapter_id)
310 + return result
311 +
312 + def remove_oldest_adapter(self) -> bool:
313 + if len(self._registered_adapters) > 0:
314 + self._registered_adapters.remove_oldest()
315 + return True
316 + return False
317 +
318 + def pin_adapter(self, prompt_adapter_id: int) -> bool:
319 + """Pin a PromptAdapterModel in the manager cache."""
320 + self._pin_prompt_adapter_in_cpu_cache(prompt_adapter_id)
321 + self._pin_prompt_adapter_in_gpu_cache(prompt_adapter_id)
322 + return True
323 +
324 + def _pin_prompt_adapter_in_cpu_cache(self, prompt_adapter_id: int):
325 + try:
326 + self._registered_adapters.pin(prompt_adapter_id)
327 + except ValueError as err:
328 + raise ValueError(
329 + "Pinning failed. "
330 + f"Prompt Adapter {prompt_adapter_id} is not registered."
331 + ) from err
332 +
333 + def _pin_prompt_adapter_in_gpu_cache(self, prompt_adapter_id: int):
334 + if prompt_adapter_id not in self._active_adapters:
335 + # move adapter to gpu if not already active
336 + self.activate_adapter(prompt_adapter_id)
337 + self._active_adapters.pin(prompt_adapter_id)
338 +
339 +
340 +def create_prompt_adapter_manager(
341 + model: nn.Module,
342 + max_num_seqs: int,
343 + max_num_batched_tokens: int,
344 + prompt_adapter_config: PromptAdapterConfig,
345 + prompt_adapter_manager_cls: Type[
346 + PromptAdapterModelManager] = PromptAdapterModelManager,
347 + **kwargs) -> PromptAdapterModelManager:
348 + """Create a PromptAdapterModel for a given model."""
349 + prompt_adapter_manager = prompt_adapter_manager_cls(
350 + model=model,
351 + max_num_seqs=max_num_seqs,
352 + max_num_batched_tokens=max_num_batched_tokens,
353 + prompt_adapter_config=prompt_adapter_config,
354 + **kwargs)
355 + return prompt_adapter_manager
vllm/prompt_adapter/request.py new
+30
@@ -0,0 +1,30 @@
1 +from dataclasses import dataclass
2 +
3 +from vllm.adapter_commons.request import AdapterRequest
4 +
5 +
6 +@dataclass
7 +class PromptAdapterRequest(AdapterRequest):
8 + """
9 + Request for a Prompt adapter.
10 + """
11 +
12 + prompt_adapter_name: str
13 + prompt_adapter_id: int
14 + prompt_adapter_local_path: str
15 + prompt_adapter_num_virtual_tokens: int
16 +
17 + def __hash__(self):
18 + return super().__hash__()
19 +
20 + @property
21 + def adapter_id(self):
22 + return self.prompt_adapter_id
23 +
24 + @property
25 + def name(self):
26 + return self.prompt_adapter_name
27 +
28 + @property
29 + def local_path(self):
30 + return self.prompt_adapter_local_path
vllm/prompt_adapter/worker_manager.py new
+176
@@ -0,0 +1,176 @@
1 +import logging
2 +from typing import Any, Optional, Set, Type
3 +
4 +import torch
5 +
6 +from vllm.adapter_commons.utils import (add_adapter_worker,
7 + apply_adapters_worker,
8 + list_adapters_worker,
9 + set_active_adapters_worker)
10 +from vllm.adapter_commons.worker_manager import AbstractWorkerManager
11 +from vllm.config import PromptAdapterConfig
12 +from vllm.prompt_adapter.models import (LRUCachePromptAdapterModelManager,
13 + PromptAdapterModel,
14 + PromptAdapterModelManager,
15 + create_prompt_adapter_manager)
16 +from vllm.prompt_adapter.request import PromptAdapterRequest
17 +
18 +logger = logging.getLogger(__name__)
19 +
20 +
21 +class WorkerPromptAdapterManager(AbstractWorkerManager):
22 + """WorkerPromptAdapterManager that manages
23 + prompt_adapter models on the worker side.
24 +
25 + Every request, the requested prompt_adapters will be
26 + loaded (unless they are already loaded),
27 + and every other prompt_adapter will be unloaded."""
28 +
29 + _manager_cls: Type[PromptAdapterModelManager] = PromptAdapterModelManager
30 +
31 + def __init__(
32 + self,
33 + max_num_seqs: int,
34 + max_num_batched_tokens: int,
35 + device: torch.device,
36 + prompt_adapter_config: PromptAdapterConfig,
37 + prompt_adapter_model_cls: Type[PromptAdapterModel] = PromptAdapterModel
38 + ):
39 + self._adapter_manager: PromptAdapterModelManager
40 + self.max_num_seqs = max_num_seqs
41 + self.max_num_batched_tokens = max_num_batched_tokens
42 + self._prompt_adapter_model_cls = prompt_adapter_model_cls
43 + self.prompt_adapter_config = prompt_adapter_config
44 + super().__init__(device)
45 +
46 + @property
47 + def is_enabled(self) -> bool:
48 + return True
49 +
50 + def create_prompt_adapter_manager(
51 + self,
52 + model: torch.nn.Module,
53 + ) -> Any:
54 + prompt_adapter_manager = create_prompt_adapter_manager(
55 + model,
56 + max_num_seqs=self.max_num_seqs,
57 + max_num_batched_tokens=self.max_num_batched_tokens,
58 + prompt_adapter_config=self.prompt_adapter_config,
59 + prompt_adapter_manager_cls=self._manager_cls,
60 + )
61 + self._adapter_manager = prompt_adapter_manager
62 + return prompt_adapter_manager.model
63 +
64 + def _load_adapter(
65 + self, prompt_adapter_request: PromptAdapterRequest
66 + ) -> PromptAdapterModel:
67 + try:
68 + prompt_adapter = (
69 + self._prompt_adapter_model_cls.from_local_checkpoint(
70 + prompt_adapter_request.prompt_adapter_local_path,
71 + prompt_adapter_id=prompt_adapter_request.prompt_adapter_id,
72 + num_virtual_tokens=prompt_adapter_request.
73 + prompt_adapter_num_virtual_tokens,
74 + config=self.prompt_adapter_config,
75 + device=str(self.device),
76 + ))
77 + except Exception as e:
78 + raise RuntimeError(
79 + f"Loading prompt_adapter "
80 + f"{prompt_adapter_request.prompt_adapter_local_path}"
81 + f" failed") from e
82 + return prompt_adapter
83 +
84 + def add_dummy_prompt_adapter(
85 + self, prompt_adapter_request: PromptAdapterRequest) -> bool:
86 + return True
87 +
88 + def pin_adapter(self, adapter_id: int) -> bool:
89 + return self._adapter_manager.pin_adapter(adapter_id)
90 +
91 + def set_active_adapters(self, requests: Set[Any],
92 + mapping: Optional[Any]) -> None:
93 + set_active_adapters_worker(requests, mapping, self._apply_adapters,
94 + self._adapter_manager.set_adapter_mapping)
95 +
96 + def add_adapter(self, adapter_request: Any) -> bool:
97 + return add_adapter_worker(adapter_request, self.list_adapters,
98 + self._load_adapter,
99 + self._adapter_manager.add_adapter,
100 + self._adapter_manager.activate_adapter)
101 +
102 + def _apply_adapters(self, adapter_requests: Set[Any]) -> None:
103 + apply_adapters_worker(adapter_requests, self.list_adapters,
104 + self._adapter_manager.adapter_slots,
105 + self.remove_adapter, self.add_adapter)
106 +
107 + def remove_adapter(self, adapter_id: int) -> bool:
108 + return self._adapter_manager.remove_adapter(adapter_id)
109 +
110 + def remove_all_adapters(self):
111 + self._adapter_manager.remove_all_adapters()
112 +
113 + def list_adapters(self) -> Set[int]:
114 + return list_adapters_worker(self._adapter_manager.list_adapters)
115 +
116 +
117 +class LRUCacheWorkerPromptAdapterManager(WorkerPromptAdapterManager):
118 + """WorkerPromptAdapterManager that manages
119 + prompt_adapter models on the worker side.
120 +
121 + Uses an LRU Cache. Every request, the requested
122 + prompt_adapters will be loaded (unless they are already loaded)
123 + and least recently used prompt_adapters will
124 + be unloaded if the cache is above capacity."""
125 +
126 + _prompt_adapter_manager_cls: Type[
127 + LRUCachePromptAdapterModelManager] = LRUCachePromptAdapterModelManager
128 +
129 + def create_prompt_adapter_manager(
130 + self,
131 + model: torch.nn.Module,
132 + ) -> Any:
133 + prompt_adapter_manager = create_prompt_adapter_manager(
134 + model,
135 + max_num_seqs=self.max_num_seqs,
136 + max_num_batched_tokens=self.max_num_batched_tokens,
137 + prompt_adapter_config=self.prompt_adapter_config,
138 + prompt_adapter_manager_cls=self._prompt_adapter_manager_cls)
139 + self._adapter_manager: LRUCachePromptAdapterModelManager = (
140 + prompt_adapter_manager)
141 + return prompt_adapter_manager.model
142 +
143 + def _apply_adapters(
144 + self, prompt_adapter_requests: Set[PromptAdapterRequest]) -> None:
145 + prompt_adapters_map = {
146 + prompt_adapter_request.prompt_adapter_id: prompt_adapter_request
147 + for prompt_adapter_request in prompt_adapter_requests
148 + if prompt_adapter_request
149 + }
150 + if len(prompt_adapters_map
151 + ) > self._adapter_manager.prompt_adapter_slots:
152 + raise RuntimeError(
153 + f"Number of requested prompt_adapters "
154 + f"({len(prompt_adapters_map)}) is greater "
155 + "than the number of GPU prompt_adapter slots "
156 + f"({self._adapter_manager.prompt_adapter_slots}).")
157 + for prompt_adapter in prompt_adapters_map.values():
158 + self.add_adapter(prompt_adapter)
159 +
160 + def add_adapter(self,
161 + prompt_adapter_request: PromptAdapterRequest) -> bool:
162 + if prompt_adapter_request.prompt_adapter_id not in self.list_adapters(
163 + ):
164 + # Remove before we load the new prompt_adapter to save memory
165 + if len(self._adapter_manager) + 1 > self._adapter_manager.capacity:
166 + self._adapter_manager.remove_oldest_adapter()
167 + prompt_adapter = self._load_adapter(prompt_adapter_request)
168 + loaded = self._adapter_manager.add_adapter(prompt_adapter)
169 + else:
170 + # If the prompt_adapter is already loaded, just touch it to
171 + # update its position in the caches
172 + loaded = self._adapter_manager.get_adapter(
173 + prompt_adapter_request.prompt_adapter_id) is not None
174 + self._adapter_manager.activate_adapter(
175 + prompt_adapter_request.prompt_adapter_id)
176 + return loaded
vllm/sequence.py
+42 -6
@@ -10,6 +10,7 @@ import torch
10
11 from vllm.lora.request import LoRARequest
12 from vllm.pooling_params import PoolingParams
13 +from vllm.prompt_adapter.request import PromptAdapterRequest
14 from vllm.sampling_params import SamplingParams
15
16 if TYPE_CHECKING:
@@ -238,21 +239,25 @@ class Sequence:
239 block_size: The block size of the sequence. Should be the same as the
240 block size used by the block manager and cache engine.
241 lora_request: LoRA request.
242 + prompt_adapter_request: Prompt Adapter request.
243 +
244 """
245
246 def __init__(
244 - self,
245 - seq_id: int,
246 - inputs: "LLMInputs",
247 - block_size: int,
248 - eos_token_id: Optional[int] = None,
249 - lora_request: Optional[LoRARequest] = None,
247 + self,
248 + seq_id: int,
249 + inputs: "LLMInputs",
250 + block_size: int,
251 + eos_token_id: Optional[int] = None,
252 + lora_request: Optional[LoRARequest] = None,
253 + prompt_adapter_request: Optional[PromptAdapterRequest] = None
254 ) -> None:
255 self.seq_id = seq_id
256 self.inputs = inputs
257 self.block_size = block_size
258 self.eos_token_id = eos_token_id
259 self.lora_request = lora_request
260 + self.prompt_adapter_request = prompt_adapter_request
261
262 self.data = SequenceData(self.prompt_token_ids)
263 self.output_logprobs: SampleLogprobs = []
@@ -287,6 +292,11 @@ class Sequence:
292 def lora_int_id(self) -> int:
293 return self.lora_request.lora_int_id if self.lora_request else 0
294
295 + @property
296 + def prompt_adapter_id(self) -> int:
297 + return self.prompt_adapter_request.prompt_adapter_id \
298 + if self.prompt_adapter_request else 0
299 +
300 def get_output_text_to_return(self, buffer_length: int):
301 # We return the full output text if the sequence is finished.
302 truncate = buffer_length and not self.is_finished()
@@ -414,6 +424,7 @@ class SequenceGroup:
424 encoder_seq: Optional, the single encoder sequence. Should be None
425 unless you are working with an encoder/decoder model.
426 trace_headers: OpenTelemetry trace headers.
427 + prompt_adapter_request: Prompt Adapter request.
428 """
429
430 def __init__(
@@ -427,6 +438,7 @@ class SequenceGroup:
438 pooling_params: Optional[PoolingParams] = None,
439 encoder_seq: Optional[Sequence] = None,
440 trace_headers: Optional[Dict[str, str]] = None,
441 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
442 ) -> None:
443 self.request_id = request_id
444 self.seqs_dict = {seq.seq_id: seq for seq in seqs}
@@ -441,6 +453,7 @@ class SequenceGroup:
453 self.state = SequenceGroupState()
454 self.embeddings = embeddings
455 self.pooling_params = pooling_params
456 + self.prompt_adapter_request = prompt_adapter_request
457 self.encoder_seq = encoder_seq
458 self.trace_headers = trace_headers
459
@@ -466,6 +479,16 @@ class SequenceGroup:
479 def lora_int_id(self) -> int:
480 return self.lora_request.lora_int_id if self.lora_request else 0
481
482 + @property
483 + def prompt_adapter_id(self) -> int:
484 + return self.prompt_adapter_request.prompt_adapter_id \
485 + if self.prompt_adapter_request else 0
486 +
487 + @property
488 + def prompt_adapter_num_virtual_tokens(self) -> int:
489 + return self.prompt_adapter_request.prompt_adapter_num_virtual_tokens\
490 + if self.prompt_adapter_request else 0
491 +
492 def get_last_latency(self, now: float) -> Optional[float]:
493 """Sets the last token time for Request level timings."""
494 # If still in prefill phase, raise Error.
@@ -624,6 +647,7 @@ class SequenceGroupMetadata:
647 (SequenceGroup.encoder_seq). Should be None
648 unless you are working with an encoder/decoder
649 model.
650 + prompt_adapter_request: Prompt Adapter request.
651 """
652
653 def __init__(
@@ -642,6 +666,7 @@ class SequenceGroupMetadata:
666 multi_modal_data: Optional["MultiModalDataDict"] = None,
667 encoder_seq_data: Optional[SequenceData] = None,
668 cross_block_table: Optional[List[int]] = None,
669 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
670 ) -> None:
671 self.request_id = request_id
672 self.is_prompt = is_prompt
@@ -650,6 +675,7 @@ class SequenceGroupMetadata:
675 self.block_tables = block_tables
676 self.pooling_params = pooling_params
677 self.lora_request = lora_request
678 + self.prompt_adapter_request = prompt_adapter_request
679 self.computed_block_nums = computed_block_nums
680 self.multi_modal_data = multi_modal_data
681 self.state = SequenceGroupState() if state is None else state
@@ -674,6 +700,16 @@ class SequenceGroupMetadata:
700 def lora_int_id(self) -> int:
701 return self.lora_request.lora_int_id if self.lora_request else 0
702
703 + @property
704 + def prompt_adapter_id(self) -> int:
705 + return self.prompt_adapter_request.prompt_adapter_id \
706 + if self.prompt_adapter_request else 0
707 +
708 + @property
709 + def prompt_adapter_num_virtual_tokens(self) -> int:
710 + return self.prompt_adapter_request.prompt_adapter_num_virtual_tokens \
711 + if self.prompt_adapter_request else 0
712 +
713 @property
714 def token_chunk_size(self) -> int:
715 """Return the number of tokens to be processed (chunk size)."""
vllm/spec_decode/draft_model_runner.py
+10 -1
@@ -4,7 +4,7 @@ import torch
4
5 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6 ModelConfig, MultiModalConfig, ParallelConfig,
7 - SchedulerConfig)
7 + PromptAdapterConfig, SchedulerConfig)
8 from vllm.logger import init_logger
9 from vllm.sequence import (IntermediateTensors, SamplerOutput,
10 SequenceGroupMetadata)
@@ -48,6 +48,7 @@ class TP1DraftModelRunner(ModelRunner):
48 kv_cache_dtype: Optional[str] = "auto",
49 is_driver_worker: bool = False,
50 multimodal_config: Optional[MultiModalConfig] = None,
51 + prompt_adapter_config: Optional[PromptAdapterConfig] = None,
52 return_hidden_states: bool = False,
53 ):
54 if return_hidden_states:
@@ -66,6 +67,7 @@ class TP1DraftModelRunner(ModelRunner):
67 kv_cache_dtype=kv_cache_dtype,
68 is_driver_worker=is_driver_worker,
69 multimodal_config=multimodal_config,
70 + prompt_adapter_config=prompt_adapter_config,
71 return_hidden_states=return_hidden_states,
72 )
73
@@ -136,6 +138,13 @@ class TP1DraftModelRunner(ModelRunner):
138 self.set_active_loras(model_input.lora_requests,
139 model_input.lora_mapping)
140
141 + if self.prompt_adapter_config:
142 + assert model_input.prompt_adapter_requests is not None
143 + assert model_input.prompt_adapter_mapping is not None
144 + self.set_active_prompt_adapters(
145 + model_input.prompt_adapter_requests,
146 + model_input.prompt_adapter_mapping)
147 +
148 virtual_engine = model_input.virtual_engine
149 outputs: List[SamplerOutput] = []
150 for step in range(num_steps):
vllm/worker/cpu_model_runner.py
+3 -1
@@ -8,7 +8,7 @@ from torch import nn
8 from vllm.attention import AttentionMetadata, get_attn_backend
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 ModelConfig, MultiModalConfig, ParallelConfig,
11 - SchedulerConfig)
11 + PromptAdapterConfig, SchedulerConfig)
12 from vllm.logger import init_logger
13 from vllm.model_executor import SamplingMetadata
14 from vllm.model_executor.model_loader import get_model
@@ -81,6 +81,7 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
81 lora_config: Optional[LoRAConfig],
82 multimodal_config: Optional[MultiModalConfig],
83 kv_cache_dtype: Optional[str] = "auto",
84 + prompt_adapter_config: Optional[PromptAdapterConfig] = None,
85 is_driver_worker: bool = False,
86 *args,
87 **kwargs,
@@ -94,6 +95,7 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
95 self.cache_config = cache_config
96 self.lora_config = lora_config
97 self.multimodal_config = multimodal_config
98 + self.prompt_adapter_config = prompt_adapter_config
99 self.load_config = load_config
100 self.is_driver_worker = is_driver_worker
101
vllm/worker/cpu_worker.py
+4 -1
@@ -7,7 +7,7 @@ import torch.distributed
7 from vllm.attention import get_attn_backend
8 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
9 ModelConfig, MultiModalConfig, ParallelConfig,
10 - SchedulerConfig)
10 + PromptAdapterConfig, SchedulerConfig)
11 from vllm.distributed import (ensure_model_parallel_initialized,
12 init_distributed_environment)
13 from vllm.logger import init_logger
@@ -133,6 +133,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
133 lora_config: Optional[LoRAConfig] = None,
134 multimodal_config: Optional[MultiModalConfig] = None,
135 kv_cache_dtype: Optional[str] = "auto",
136 + prompt_adapter_config: Optional[PromptAdapterConfig] = None,
137 is_driver_worker: bool = False,
138 ) -> None:
139 self.model_config = model_config
@@ -145,6 +146,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
146 self.rank = rank
147 self.distributed_init_method = distributed_init_method
148 self.lora_config = lora_config
149 + self.prompt_adapter_config = prompt_adapter_config
150 self.multimodal_config = multimodal_config
151 self.is_driver_worker = is_driver_worker
152 if self.is_driver_worker:
@@ -167,6 +169,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
169 lora_config=self.lora_config,
170 multimodal_config=self.multimodal_config,
171 kv_cache_dtype=kv_cache_dtype,
172 + prompt_adapter_config=self.prompt_adapter_config,
173 is_driver_worker=is_driver_worker)
174 # Uninitialized cache engine. Will be initialized by
175 # initialize_cache.
vllm/worker/embedding_model_runner.py
+10 -1
@@ -5,7 +5,7 @@ import torch
5
6 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7 ModelConfig, MultiModalConfig, ParallelConfig,
8 - SchedulerConfig)
8 + PromptAdapterConfig, SchedulerConfig)
9 from vllm.logger import init_logger
10 from vllm.model_executor.pooling_metadata import PoolingMetadata
11 from vllm.pooling_params import PoolingParams
@@ -40,6 +40,7 @@ class EmbeddingModelRunner(
40 lora_config: Optional[LoRAConfig],
41 kv_cache_dtype: Optional[str] = "auto",
42 is_driver_worker: bool = False,
43 + prompt_adapter_config: Optional[PromptAdapterConfig] = None,
44 multimodal_config: Optional[MultiModalConfig] = None,
45 ):
46 super().__init__(model_config,
@@ -51,6 +52,7 @@ class EmbeddingModelRunner(
52 lora_config=lora_config,
53 kv_cache_dtype=kv_cache_dtype,
54 is_driver_worker=is_driver_worker,
55 + prompt_adapter_config=prompt_adapter_config,
56 multimodal_config=multimodal_config)
57
58 @torch.inference_mode()
@@ -71,6 +73,13 @@ class EmbeddingModelRunner(
73 self.set_active_loras(model_input.lora_requests,
74 model_input.lora_mapping)
75
76 + if self.prompt_adapter_config:
77 + assert model_input.prompt_adapter_requests is not None
78 + assert model_input.prompt_adapter_mapping is not None
79 + self.set_active_prompt_adapters(
80 + model_input.prompt_adapter_requests,
81 + model_input.prompt_adapter_mapping)
82 +
83 # Currently cuda graph is only supported by the decode phase.
84 assert model_input.attn_metadata is not None
85 prefill_meta = model_input.attn_metadata.prefill_metadata
vllm/worker/model_runner.py
+118 -20
@@ -25,7 +25,7 @@ except ImportError:
25 from vllm.attention import AttentionMetadata, get_attn_backend
26 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
27 ModelConfig, MultiModalConfig, ParallelConfig,
28 - SchedulerConfig)
28 + PromptAdapterConfig, SchedulerConfig)
29 from vllm.distributed import get_pp_group
30 from vllm.distributed.parallel_state import graph_capture
31 from vllm.inputs import INPUT_REGISTRY
@@ -40,6 +40,10 @@ from vllm.model_executor.models.interfaces import (supports_lora,
40 supports_vision)
41 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensors,
42 MultiModalInputs)
43 +from vllm.prompt_adapter.layers import PromptAdapterMapping
44 +from vllm.prompt_adapter.request import PromptAdapterRequest
45 +from vllm.prompt_adapter.worker_manager import (
46 + LRUCacheWorkerPromptAdapterManager)
47 from vllm.sampling_params import SamplingParams
48 from vllm.sequence import (IntermediateTensors, SamplerOutput,
49 SequenceGroupMetadata)
@@ -85,6 +89,8 @@ class ModelInputForGPU(ModelRunnerInputBase):
89 lora_mapping: Optional["LoRAMapping"] = None
90 lora_requests: Optional[Set[LoRARequest]] = None
91 attn_metadata: Optional["AttentionMetadata"] = None
92 + prompt_adapter_mapping: Optional[PromptAdapterMapping] = None
93 + prompt_adapter_requests: Optional[Set[PromptAdapterRequest]] = None
94 multi_modal_kwargs: Optional[Mapping[str, BatchedTensors]] = None
95 request_ids_to_seq_ids: Optional[Dict[str, List[int]]] = None
96 finished_requests_ids: Optional[List[str]] = None
@@ -97,6 +103,8 @@ class ModelInputForGPU(ModelRunnerInputBase):
103 "lora_requests": self.lora_requests,
104 "lora_mapping": self.lora_mapping,
105 "multi_modal_kwargs": self.multi_modal_kwargs,
106 + "prompt_adapter_mapping": self.prompt_adapter_mapping,
107 + "prompt_adapter_requests": self.prompt_adapter_requests,
108 "virtual_engine": self.virtual_engine,
109 "request_ids_to_seq_ids": self.request_ids_to_seq_ids,
110 "finished_requests_ids": self.finished_requests_ids,
@@ -133,6 +141,8 @@ class ModelInputForGPUWithSamplingMetadata(ModelInputForGPU):
141 "lora_requests": self.lora_requests,
142 "lora_mapping": self.lora_mapping,
143 "multi_modal_kwargs": self.multi_modal_kwargs,
144 + "prompt_adapter_mapping": self.prompt_adapter_mapping,
145 + "prompt_adapter_requests": self.prompt_adapter_requests,
146 "virtual_engine": self.virtual_engine,
147 "request_ids_to_seq_ids": self.request_ids_to_seq_ids,
148 "finished_requests_ids": self.finished_requests_ids,
@@ -172,6 +182,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
182 lora_config: Optional[LoRAConfig],
183 kv_cache_dtype: Optional[str] = "auto",
184 is_driver_worker: bool = False,
185 + prompt_adapter_config: Optional[PromptAdapterConfig] = None,
186 multimodal_config: Optional[MultiModalConfig] = None,
187 return_hidden_states: bool = False,
188 ):
@@ -183,6 +194,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
194 self.lora_config = lora_config
195 self.load_config = load_config
196 self.is_driver_worker = is_driver_worker
197 + self.prompt_adapter_config = prompt_adapter_config
198 self.multimodal_config = multimodal_config
199 self.return_hidden_states = return_hidden_states
200
@@ -232,6 +244,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
244 self.model: nn.Module # Set after load_model
245 # Set after load_model.
246 self.lora_manager: Optional[LRUCacheWorkerLoRAManager] = None
247 + self.prompt_adapter_manager: LRUCacheWorkerPromptAdapterManager = None
248
249 self.flashinfer_decode_workspace_buffer = None
250 self.flashinfer_decode_wrapper = None
@@ -240,16 +253,14 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
253
254 def load_model(self) -> None:
255 with CudaMemoryProfiler() as m:
243 - self.model = get_model(
244 - model_config=self.model_config,
245 - device_config=self.device_config,
246 - load_config=self.load_config,
247 - lora_config=self.lora_config,
248 - multimodal_config=self.multimodal_config,
249 - parallel_config=self.parallel_config,
250 - scheduler_config=self.scheduler_config,
251 - cache_config=self.cache_config,
252 - )
256 + self.model = get_model(model_config=self.model_config,
257 + device_config=self.device_config,
258 + load_config=self.load_config,
259 + lora_config=self.lora_config,
260 + multimodal_config=self.multimodal_config,
261 + parallel_config=self.parallel_config,
262 + scheduler_config=self.scheduler_config,
263 + cache_config=self.cache_config)
264
265 self.model_memory_usage = m.consumed_memory
266 logger.info("Loading model weights took %.4f GB",
@@ -274,6 +285,15 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
285 )
286 self.model = self.lora_manager.create_lora_manager(self.model)
287
288 + if self.prompt_adapter_config:
289 + self.prompt_adapter_manager = LRUCacheWorkerPromptAdapterManager(
290 + self.scheduler_config.max_num_seqs,
291 + self.scheduler_config.max_num_batched_tokens, self.device,
292 + self.prompt_adapter_config)
293 + self.model = (
294 + self.prompt_adapter_manager.create_prompt_adapter_manager(
295 + self.model))
296 +
297 if self.kv_cache_dtype == "fp8" and is_hip():
298 # Currently only ROCm accepts kv-cache scaling factors
299 # via quantization_param_path and this will be deprecated
@@ -354,6 +374,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
374 lora_index_mapping: List[int] = []
375 lora_prompt_mapping: List[int] = []
376 lora_requests: Set[LoRARequest] = set()
377 + prompt_adapter_index_mapping: List[int] = []
378 + prompt_adapter_prompt_mapping: List[int] = []
379 + prompt_adapter_requests: Set[PromptAdapterRequest] = set()
380
381 seq_lens: List[int] = []
382 prefill_seq_lens: List[int] = []
@@ -504,6 +527,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
527 input_tokens.extend(tokens)
528 input_positions.extend(list(range(context_len, seq_len)))
529 lora_id = seq_group_metadata.lora_int_id
530 + prompt_adapter_id = seq_group_metadata.prompt_adapter_id
531
532 if is_prompt:
533 assert len(seq_ids) == 1
@@ -534,6 +558,21 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
558 mm_kwargs = self.multi_modal_input_mapper(mm_data)
559 multi_modal_inputs_list.append(mm_kwargs)
560
561 + if prompt_adapter_id > 0 and is_prompt:
562 + prompt_adapter_requests.add(
563 + seq_group_metadata.prompt_adapter_request)
564 +
565 + num_tokens = seq_group_metadata.\
566 + prompt_adapter_num_virtual_tokens
567 + pm = [prompt_adapter_id
568 + ] * num_tokens + [0] * (query_len - num_tokens)
569 + prompt_adapter_index_mapping += pm
570 + prompt_adapter_prompt_mapping.extend(
571 + [prompt_adapter_id] *
572 + (query_len if seq_group_metadata.sampling_params
573 + and seq_group_metadata.sampling_params.prompt_logprobs
574 + else 1))
575 +
576 is_profile_run = _is_block_tables_empty(
577 seq_group_metadata.block_tables)
578 if is_profile_run:
@@ -618,12 +657,11 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
657 seq_lens.append(1)
658 block_tables.append([])
659 lora_index_mapping.append(0)
621 -
660 + prompt_adapter_index_mapping.append(0)
661 if self.attn_backend.get_name() == "flashinfer":
662 last_paged_kv_indptr = paged_kv_indptr[-1]
663 paged_kv_indptr.append(last_paged_kv_indptr)
664 paged_kv_last_page_len.append(0)
626 -
665 batch_size = graph_batch_size
666 num_decode_tokens = batch_size
667
@@ -759,6 +797,14 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
797 else:
798 lora_mapping = None
799
800 + if self.prompt_adapter_config:
801 + prompt_adapter_mapping = PromptAdapterMapping(
802 + prompt_adapter_index_mapping,
803 + prompt_adapter_prompt_mapping,
804 + )
805 + else:
806 + prompt_adapter_mapping = None
807 +
808 multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list,
809 device=self.device)
810 request_ids_to_seq_ids = {
@@ -776,7 +822,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
822 lora_requests=lora_requests,
823 multi_modal_kwargs=multi_modal_kwargs,
824 request_ids_to_seq_ids=request_ids_to_seq_ids,
779 - finished_requests_ids=finished_requests_ids)
825 + finished_requests_ids=finished_requests_ids,
826 + prompt_adapter_mapping=prompt_adapter_mapping,
827 + prompt_adapter_requests=prompt_adapter_requests,
828 + )
829
830 @torch.inference_mode()
831 def profile_run(self) -> None:
@@ -878,33 +927,67 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
927 def remove_all_loras(self):
928 if not self.lora_manager:
929 raise RuntimeError("LoRA is not enabled.")
881 - self.lora_manager.remove_all_loras()
930 + self.lora_manager.remove_all_adapters()
931
932 def set_active_loras(self, lora_requests: Set[LoRARequest],
933 lora_mapping: LoRAMapping) -> None:
934 if not self.lora_manager:
935 raise RuntimeError("LoRA is not enabled.")
887 - self.lora_manager.set_active_loras(lora_requests, lora_mapping)
936 + self.lora_manager.set_active_adapters(lora_requests, lora_mapping)
937
938 def add_lora(self, lora_request: LoRARequest) -> bool:
939 if not self.lora_manager:
940 raise RuntimeError("LoRA is not enabled.")
892 - return self.lora_manager.add_lora(lora_request)
941 + return self.lora_manager.add_adapter(lora_request)
942
943 def remove_lora(self, lora_id: int) -> bool:
944 if not self.lora_manager:
945 raise RuntimeError("LoRA is not enabled.")
897 - return self.lora_manager.remove_lora(lora_id)
946 + return self.lora_manager.remove_adapter(lora_id)
947
948 def pin_lora(self, lora_id: int) -> bool:
949 if not self.lora_manager:
950 raise RuntimeError("LoRA is not enabled.")
902 - return self.lora_manager.pin_lora(lora_id)
951 + return self.lora_manager.pin_adapter(lora_id)
952
953 def list_loras(self) -> Set[int]:
954 if not self.lora_manager:
955 raise RuntimeError("LoRA is not enabled.")
907 - return self.lora_manager.list_loras()
956 + return self.lora_manager.list_adapters()
957 +
958 + def remove_all_prompt_adapters(self):
959 + if not self.prompt_adapter_manager:
960 + raise RuntimeError("PromptAdapter is not enabled.")
961 + self.prompt_adapter_manager.remove_all_adapters()
962 +
963 + def set_active_prompt_adapters(
964 + self, prompt_adapter_requests: Set[PromptAdapterRequest],
965 + prompt_adapter_mapping: PromptAdapterMapping) -> None:
966 + if not self.prompt_adapter_manager:
967 + raise RuntimeError("PromptAdapter is not enabled.")
968 + self.prompt_adapter_manager.set_active_adapters(
969 + prompt_adapter_requests, prompt_adapter_mapping)
970 +
971 + def add_prompt_adapter(
972 + self, prompt_adapter_request: PromptAdapterRequest) -> bool:
973 + if not self.prompt_adapter_manager:
974 + raise RuntimeError("PromptAdapter is not enabled.")
975 + return self.prompt_adapter_manager.add_adapter(prompt_adapter_request)
976 +
977 + def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
978 + if not self.prompt_adapter_manager:
979 + raise RuntimeError("PromptAdapter is not enabled.")
980 + return self.prompt_adapter_manager.remove_adapter(prompt_adapter_id)
981 +
982 + def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
983 + if not self.prompt_adapter_manager:
984 + raise RuntimeError("PromptAdapter is not enabled.")
985 + return self.prompt_adapter_manager.pin_adapter(prompt_adapter_id)
986 +
987 + def list_prompt_adapters(self) -> Set[int]:
988 + if not self.prompt_adapter_manager:
989 + raise RuntimeError("PromptAdapter is not enabled.")
990 + return self.prompt_adapter_manager.list_adapters()
991
992 @torch.inference_mode()
993 def capture_model(self, kv_caches: List[List[torch.Tensor]]) -> None:
@@ -1063,6 +1146,14 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1146 )
1147 self.set_active_loras(set(), lora_mapping)
1148
1149 + if self.prompt_adapter_config:
1150 + prompt_adapter_mapping = PromptAdapterMapping(
1151 + [-1] * batch_size,
1152 + [-1] * batch_size,
1153 + )
1154 + self.set_active_prompt_adapters(
1155 + set(), prompt_adapter_mapping)
1156 +
1157 graph_runner = CUDAGraphRunner(
1158 self.model, self.attn_backend.get_name())
1159
@@ -1189,6 +1280,13 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1280 self.set_active_loras(model_input.lora_requests,
1281 model_input.lora_mapping)
1282
1283 + if self.prompt_adapter_config:
1284 + assert model_input.prompt_adapter_requests is not None
1285 + assert model_input.prompt_adapter_mapping is not None
1286 + self.set_active_prompt_adapters(
1287 + model_input.prompt_adapter_requests,
1288 + model_input.prompt_adapter_mapping)
1289 +
1290 if self.attn_backend.get_name() == "flashinfer":
1291 assert model_input.attn_metadata is not None
1292 assert model_input.input_tokens is not None
vllm/worker/worker.py
+19 -1
@@ -8,7 +8,8 @@ import torch.distributed
8
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 ModelConfig, MultiModalConfig, ParallelConfig,
11 - SchedulerConfig, SpeculativeConfig)
11 + PromptAdapterConfig, SchedulerConfig,
12 + SpeculativeConfig)
13 from vllm.distributed import (ensure_model_parallel_initialized,
14 init_distributed_environment,
15 set_custom_all_reduce)
@@ -16,6 +17,7 @@ from vllm.lora.request import LoRARequest
17 from vllm.model_executor import set_random_seed
18 from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
19 from vllm.platforms import current_platform
20 +from vllm.prompt_adapter.request import PromptAdapterRequest
21 from vllm.sequence import ExecuteModelRequest
22 from vllm.worker.cache_engine import CacheEngine
23 from vllm.worker.embedding_model_runner import EmbeddingModelRunner
@@ -45,6 +47,7 @@ class Worker(LocalOrDistributedWorkerBase):
47 lora_config: Optional[LoRAConfig] = None,
48 multimodal_config: Optional[MultiModalConfig] = None,
49 speculative_config: Optional[SpeculativeConfig] = None,
50 + prompt_adapter_config: Optional[PromptAdapterConfig] = None,
51 is_driver_worker: bool = False,
52 model_runner_cls: Optional[Type[GPUModelRunnerBase]] = None,
53 ) -> None:
@@ -59,6 +62,7 @@ class Worker(LocalOrDistributedWorkerBase):
62 self.distributed_init_method = distributed_init_method
63 self.lora_config = lora_config
64 self.load_config = load_config
65 + self.prompt_adapter_config = prompt_adapter_config
66 self.is_driver_worker = is_driver_worker
67 if parallel_config and is_driver_worker:
68 assert rank % parallel_config.tensor_parallel_size == 0, \
@@ -92,6 +96,7 @@ class Worker(LocalOrDistributedWorkerBase):
96 lora_config=self.lora_config,
97 kv_cache_dtype=self.cache_config.cache_dtype,
98 is_driver_worker=is_driver_worker,
99 + prompt_adapter_config=prompt_adapter_config,
100 multimodal_config=multimodal_config,
101 **speculative_args,
102 )
@@ -296,6 +301,19 @@ class Worker(LocalOrDistributedWorkerBase):
301 def list_loras(self) -> Set[int]:
302 return self.model_runner.list_loras()
303
304 + def add_prompt_adapter(
305 + self, prompt_adapter_request: PromptAdapterRequest) -> bool:
306 + return self.model_runner.add_prompt_adapter(prompt_adapter_request)
307 +
308 + def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
309 + return self.model_runner.remove_lora(prompt_adapter_id)
310 +
311 + def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
312 + return self.model_runner.pin_prompt_adapter(prompt_adapter_id)
313 +
314 + def list_prompt_adapters(self) -> Set[int]:
315 + return self.model_runner.list_prompt_adapters()
316 +
317 @property
318 def max_model_len(self) -> int:
319 return self.model_config.max_model_len
vllm/worker/xpu_model_runner.py
+3 -1
@@ -8,7 +8,7 @@ import torch.nn as nn
8 from vllm.attention import get_attn_backend
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 ModelConfig, MultiModalConfig, ParallelConfig,
11 - SchedulerConfig)
11 + PromptAdapterConfig, SchedulerConfig)
12 from vllm.distributed import broadcast_tensor_dict
13 from vllm.inputs import INPUT_REGISTRY
14 from vllm.logger import init_logger
@@ -88,6 +88,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
88 lora_config: Optional[LoRAConfig],
89 multimodal_config: Optional[MultiModalConfig],
90 kv_cache_dtype: Optional[str] = "auto",
91 + prompt_adapter_config: Optional[PromptAdapterConfig] = None,
92 is_driver_worker: bool = False,
93 *args,
94 **kwargs,
@@ -98,6 +99,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
99 self.lora_config = lora_config
100 self.load_config = load_config
101 self.cache_config = cache_config
102 + self.prompt_adapter_config = prompt_adapter_config
103 self.multimodal_config = multimodal_config
104 self.is_driver_worker = is_driver_worker
105
vllm/worker/xpu_worker.py
+4 -1
@@ -10,7 +10,8 @@ import torch.distributed
10
11 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12 ModelConfig, MultiModalConfig, ParallelConfig,
13 - SchedulerConfig, SpeculativeConfig)
13 + PromptAdapterConfig, SchedulerConfig,
14 + SpeculativeConfig)
15 from vllm.distributed import (ensure_model_parallel_initialized,
16 init_distributed_environment)
17 from vllm.logger import init_logger
@@ -47,6 +48,7 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
48 lora_config: Optional[LoRAConfig] = None,
49 multimodal_config: Optional[MultiModalConfig] = None,
50 speculative_config: Optional[SpeculativeConfig] = None,
51 + prompt_adapter_config: Optional[PromptAdapterConfig] = None,
52 is_driver_worker: bool = False,
53 ) -> None:
54 assert device_config.device_type == "xpu"
@@ -63,6 +65,7 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
65 self.rank = rank
66 self.distributed_init_method = distributed_init_method
67 self.lora_config = lora_config
68 + self.prompt_adapter_config = prompt_adapter_config
69 self.is_driver_worker = is_driver_worker
70 if self.is_driver_worker:
71 assert self.rank == 0, "The driver worker must have rank 0."