A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[CORE] Adding support for insertion of soft-tuned prompts (#4645)
Co-authored-by: Swapnil Parekh <swapnilp@ibm.com> Co-authored-by: Joe G <joseph.granados@h2o.ai> Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>
Swapnil Parekh committed
Jul 9, 2024 at 16:26 UTC
4d6ada947c7e6379b6857bc9a9a1203679d32039
48 files changed
+1951
-518
format.sh
+1
@@ -111,6 +111,7 @@ mypy vllm/spec_decode --config-file pyproject.toml
111
mypy vllm/model_executor --config-file pyproject.toml
112
mypy vllm/lora --config-file pyproject.toml
113
mypy vllm/logging --config-file pyproject.toml
114
+mypy vllm/prompt_adapter --config-file pyproject.toml
115
mypy tests --config-file pyproject.toml
116
117
tests/lora/test_long_context.py
+4
-5
@@ -92,11 +92,10 @@ def batched_generate(
92
for input in inputs:
93
prompt, sampling_param, lora_req = input
94
# Add requests to the engine and run the engine
95
- llm._validate_and_add_requests(
96
- prompt,
97
- sampling_param,
98
- lora_request=lora_req,
99
- )
95
+ llm._validate_and_add_requests(prompt,
96
+ sampling_param,
97
+ lora_request=lora_req,
98
+ prompt_adapter_request=None)
99
100
outputs = llm._run_engine(use_tqdm=True)
101
return [outputs[i].outputs[0].text.strip() for i in range(len(outputs))]
tests/lora/test_lora_manager.py
+164
-162
@@ -127,37 +127,37 @@ def test_lora_model_manager(dist_init, dummy_model):
127
model, 2, 2, 2,
128
LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2))
129
assert all(x is None for x in manager.lora_index_to_id)
130
- assert manager.add_lora(model_lora1)
131
- assert manager.activate_lora(1)
130
+ assert manager.add_adapter(model_lora1)
131
+ assert manager.activate_adapter(1)
132
assert manager.lora_index_to_id[0] == 1
133
- assert not manager.add_lora(model_lora1)
134
- assert not manager.activate_lora(1)
135
- assert manager.add_lora(model_lora2)
136
- assert manager.activate_lora(2)
133
+ assert not manager.add_adapter(model_lora1)
134
+ assert not manager.activate_adapter(1)
135
+ assert manager.add_adapter(model_lora2)
136
+ assert manager.activate_adapter(2)
137
assert manager.lora_index_to_id[0] == 1
138
assert manager.lora_index_to_id[1] == 2
139
- assert not manager.add_lora(model_lora2)
140
- assert not manager.activate_lora(2)
141
- assert manager.add_lora(model_lora3)
139
+ assert not manager.add_adapter(model_lora2)
140
+ assert not manager.activate_adapter(2)
141
+ assert manager.add_adapter(model_lora3)
142
assert manager.lora_index_to_id[0] == 1
143
assert manager.lora_index_to_id[1] == 2
144
with pytest.raises(ValueError):
145
- assert manager.activate_lora(3)
145
+ assert manager.activate_adapter(3)
146
assert manager.lora_index_to_id[0] == 1
147
assert manager.lora_index_to_id[1] == 2
148
- assert manager.remove_lora(model_lora2.id)
148
+ assert manager.remove_adapter(model_lora2.id)
149
assert manager.lora_index_to_id[1] is None
150
- assert not manager.remove_lora(model_lora2.id)
151
- assert manager.remove_lora(model_lora1.id)
152
- assert not manager.remove_lora(model_lora1.id)
153
- assert manager.add_lora(model_lora1)
150
+ assert not manager.remove_adapter(model_lora2.id)
151
+ assert manager.remove_adapter(model_lora1.id)
152
+ assert not manager.remove_adapter(model_lora1.id)
153
+ assert manager.add_adapter(model_lora1)
154
assert manager.lora_index_to_id[0] is None
155
assert manager.lora_index_to_id[1] is None
156
- assert manager.add_lora(model_lora2)
157
- assert manager.activate_lora(3)
156
+ assert manager.add_adapter(model_lora2)
157
+ assert manager.activate_adapter(3)
158
assert manager.lora_index_to_id[0] == 3
159
assert manager.lora_index_to_id[1] is None
160
- assert manager.activate_lora(2)
160
+ assert manager.activate_adapter(2)
161
assert manager.lora_index_to_id[0] == 3
162
assert manager.lora_index_to_id[1] == 2
163
@@ -173,70 +173,70 @@ def test_lora_lru_cache_model_manager(dist_init, dummy_model):
173
model, 2, 2, 2,
174
LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2))
175
assert all(x is None for x in manager.lora_index_to_id)
176
- assert manager.add_lora(model_lora1)
177
- assert manager.activate_lora(1)
176
+ assert manager.add_adapter(model_lora1)
177
+ assert manager.activate_adapter(1)
178
assert manager.lora_index_to_id[0] == 1
179
- assert not manager.add_lora(model_lora1)
180
- assert not manager.activate_lora(1)
181
- assert manager.add_lora(model_lora2)
182
- assert manager.activate_lora(2)
179
+ assert not manager.add_adapter(model_lora1)
180
+ assert not manager.activate_adapter(1)
181
+ assert manager.add_adapter(model_lora2)
182
+ assert manager.activate_adapter(2)
183
assert manager.lora_index_to_id[0] == 1
184
assert manager.lora_index_to_id[1] == 2
185
- assert not manager.add_lora(model_lora2)
186
- assert not manager.activate_lora(2)
187
- assert manager.add_lora(model_lora3)
185
+ assert not manager.add_adapter(model_lora2)
186
+ assert not manager.activate_adapter(2)
187
+ assert manager.add_adapter(model_lora3)
188
assert manager.lora_index_to_id[0] == 1
189
assert manager.lora_index_to_id[1] == 2
190
- assert manager.activate_lora(3)
190
+ assert manager.activate_adapter(3)
191
assert manager.lora_index_to_id[0] == 3
192
assert manager.lora_index_to_id[1] == 2
193
- assert manager.remove_lora(model_lora2.id)
193
+ assert manager.remove_adapter(model_lora2.id)
194
assert manager.lora_index_to_id[1] is None
195
- assert not manager.remove_lora(model_lora2.id)
196
- assert manager.remove_lora(model_lora1.id)
197
- assert not manager.remove_lora(model_lora1.id)
198
- assert manager.add_lora(model_lora1)
199
- assert manager.activate_lora(1)
195
+ assert not manager.remove_adapter(model_lora2.id)
196
+ assert manager.remove_adapter(model_lora1.id)
197
+ assert not manager.remove_adapter(model_lora1.id)
198
+ assert manager.add_adapter(model_lora1)
199
+ assert manager.activate_adapter(1)
200
assert manager.lora_index_to_id[0] == 3
201
assert manager.lora_index_to_id[1] == 1
202
- assert manager.add_lora(model_lora2)
203
- assert manager.deactivate_lora(3)
202
+ assert manager.add_adapter(model_lora2)
203
+ assert manager.deactivate_adapter(3)
204
assert manager.lora_index_to_id[0] is None
205
assert manager.lora_index_to_id[1] == 1
206
- assert manager.activate_lora(2)
206
+ assert manager.activate_adapter(2)
207
assert manager.lora_index_to_id[0] == 2
208
assert manager.lora_index_to_id[1] == 1
209
- assert manager.activate_lora(3)
209
+ assert manager.activate_adapter(3)
210
assert manager.lora_index_to_id[0] == 2
211
assert manager.lora_index_to_id[1] == 3
212
- assert manager.pin_lora(2)
212
+ assert manager.pin_adapter(2)
213
assert manager.lora_index_to_id[0] == 2
214
assert manager.lora_index_to_id[1] == 3
215
- assert manager.activate_lora(1)
215
+ assert manager.activate_adapter(1)
216
assert manager.lora_index_to_id[0] == 2
217
assert manager.lora_index_to_id[1] == 1
218
- assert manager.deactivate_lora(2)
218
+ assert manager.deactivate_adapter(2)
219
assert manager.lora_index_to_id[0] is None
220
assert manager.lora_index_to_id[1] == 1
221
- assert manager.activate_lora(3)
221
+ assert manager.activate_adapter(3)
222
assert manager.lora_index_to_id[0] == 3
223
assert manager.lora_index_to_id[1] == 1
224
- assert manager.pin_lora(3)
225
- assert manager.pin_lora(1)
224
+ assert manager.pin_adapter(3)
225
+ assert manager.pin_adapter(1)
226
with pytest.raises(RuntimeError):
227
- assert manager.pin_lora(2)
227
+ assert manager.pin_adapter(2)
228
assert manager.lora_index_to_id[0] == 3
229
assert manager.lora_index_to_id[1] == 1
230
with pytest.raises(RuntimeError):
231
- assert manager.activate_lora(2)
231
+ assert manager.activate_adapter(2)
232
233
- assert manager.deactivate_lora(3)
234
- assert manager.pin_lora(2)
233
+ assert manager.deactivate_adapter(3)
234
+ assert manager.pin_adapter(2)
235
assert manager.lora_index_to_id[0] == 2
236
assert manager.lora_index_to_id[1] == 1
237
- assert manager.remove_lora(3)
237
+ assert manager.remove_adapter(3)
238
with pytest.raises(ValueError):
239
- assert manager.pin_lora(3)
239
+ assert manager.pin_adapter(3)
240
241
242
def test_lru_lora_model_manager(dist_init, dummy_model):
@@ -256,168 +256,169 @@ def test_lru_lora_model_manager(dist_init, dummy_model):
256
assert all(x is None for x in manager.lora_index_to_id)
257
258
# Add up to capacity
259
- assert manager.add_lora(model_lora1)
260
- assert manager.add_lora(model_lora2)
261
- assert manager.activate_lora(1)
262
- assert manager.activate_lora(2)
259
+ assert manager.add_adapter(model_lora1)
260
+ assert manager.add_adapter(model_lora2)
261
+ assert manager.activate_adapter(1)
262
+ assert manager.activate_adapter(2)
263
264
- assert set(manager.list_loras()) == {1, 2}
264
+ assert set(manager.list_adapters()) == {1, 2}
265
assert manager.lora_index_to_id[0] == 1
266
assert manager.lora_index_to_id[1] == 2
267
268
# Add over capacity
269
- assert manager.add_lora(model_lora3)
270
- assert manager.add_lora(model_lora4)
271
- assert manager.activate_lora(3)
272
- assert manager.activate_lora(4)
269
+ assert manager.add_adapter(model_lora3)
270
+ assert manager.add_adapter(model_lora4)
271
+ assert manager.activate_adapter(3)
272
+ assert manager.activate_adapter(4)
273
274
- assert set(manager.list_loras()) == {3, 4}
274
+ assert set(manager.list_adapters()) == {3, 4}
275
assert manager.lora_index_to_id[0] == 3
276
assert manager.lora_index_to_id[1] == 4
277
278
# Add 3 again to move it to the top and then add 2
279
# should return false since it's in already
280
- assert not manager.add_lora(model_lora3)
281
- assert not manager.activate_lora(3)
282
- assert manager.add_lora(model_lora2)
283
- assert manager.activate_lora(2)
280
+ assert not manager.add_adapter(model_lora3)
281
+ assert not manager.activate_adapter(3)
282
+ assert manager.add_adapter(model_lora2)
283
+ assert manager.activate_adapter(2)
284
285
- assert set(manager.list_loras()) == {3, 2}
285
+ assert set(manager.list_adapters()) == {3, 2}
286
assert manager.lora_index_to_id[0] == 3
287
assert manager.lora_index_to_id[1] == 2
288
289
# Remove manually
290
- assert manager.remove_lora(3)
291
- assert not manager.remove_lora(3)
290
+ assert manager.remove_adapter(3)
291
+ assert not manager.remove_adapter(3)
292
293
- assert set(manager.list_loras()) == {2}
293
+ assert set(manager.list_adapters()) == {2}
294
assert manager.lora_index_to_id[0] is None
295
assert manager.lora_index_to_id[1] == 2
296
297
- assert manager.add_lora(model_lora3)
298
- assert manager.activate_lora(3)
299
- assert manager.add_lora(model_lora4)
300
- assert manager.activate_lora(4)
297
+ assert manager.add_adapter(model_lora3)
298
+ assert manager.activate_adapter(3)
299
+ assert manager.add_adapter(model_lora4)
300
+ assert manager.activate_adapter(4)
301
302
- assert set(manager.list_loras()) == {3, 4}
302
+ assert set(manager.list_adapters()) == {3, 4}
303
assert manager.lora_index_to_id[0] == 3
304
assert manager.lora_index_to_id[1] == 4
305
306
- assert manager.remove_oldest_lora()
307
- assert set(manager.list_loras()) == {4}
306
+ assert manager.remove_oldest_adapter()
307
+ assert set(manager.list_adapters()) == {4}
308
assert manager.lora_index_to_id[0] is None
309
assert manager.lora_index_to_id[1] == 4
310
311
- assert manager.remove_oldest_lora()
312
- assert set(manager.list_loras()) == set()
311
+ assert manager.remove_oldest_adapter()
312
+ assert set(manager.list_adapters()) == set()
313
assert all(x is None for x in manager.lora_index_to_id)
314
315
- assert not manager.remove_oldest_lora()
316
- assert set(manager.list_loras()) == set()
315
+ assert not manager.remove_oldest_adapter()
316
+ assert set(manager.list_adapters()) == set()
317
assert all(x is None for x in manager.lora_index_to_id)
318
319
# pinning
320
- assert manager.add_lora(model_lora3)
321
- assert manager.activate_lora(3)
322
- assert manager.add_lora(model_lora4)
323
- assert manager.activate_lora(4)
324
- assert set(manager.list_loras()) == {3, 4}
320
+ assert manager.add_adapter(model_lora3)
321
+ assert manager.activate_adapter(3)
322
+ assert manager.add_adapter(model_lora4)
323
+ assert manager.activate_adapter(4)
324
+ assert set(manager.list_adapters()) == {3, 4}
325
with pytest.raises(ValueError):
326
- assert manager.pin_lora(1)
327
- assert manager.pin_lora(3)
326
+ assert manager.pin_adapter(1)
327
+ assert manager.pin_adapter(3)
328
# Remove manually
329
- assert manager.remove_lora(3)
330
- assert not manager.remove_lora(3)
329
+ assert manager.remove_adapter(3)
330
+ assert not manager.remove_adapter(3)
331
332
- assert set(manager.list_loras()) == {4}
332
+ assert set(manager.list_adapters()) == {4}
333
assert manager.lora_index_to_id[0] is None
334
assert manager.lora_index_to_id[1] == 4
335
336
- assert manager.add_lora(model_lora1)
337
- assert manager.pin_lora(1)
338
- assert manager.add_lora(model_lora2)
339
- assert manager.activate_lora(2)
336
+ assert manager.add_adapter(model_lora1)
337
+ assert manager.pin_adapter(1)
338
+ assert manager.add_adapter(model_lora2)
339
+ assert manager.activate_adapter(2)
340
341
- assert set(manager.list_loras()) == {1, 2}
341
+ assert set(manager.list_adapters()) == {1, 2}
342
assert manager.lora_index_to_id[0] == 1
343
assert manager.lora_index_to_id[1] == 2
344
345
- assert manager.remove_oldest_lora()
346
- assert set(manager.list_loras()) == {1}
345
+ assert manager.remove_oldest_adapter()
346
+ assert set(manager.list_adapters()) == {1}
347
assert manager.lora_index_to_id[0] == 1
348
assert manager.lora_index_to_id[1] is None
349
350
with pytest.raises(RuntimeError):
351
- assert manager.remove_oldest_lora()
351
+ assert manager.remove_oldest_adapter()
352
353
- assert set(manager.list_loras()) == {1}
353
+ assert set(manager.list_adapters()) == {1}
354
355
356
-def test_lru_cache_worker_lora_manager(llama_2_7b_model_extra_embeddings,
357
- sql_lora_files):
356
+def test_lru_cache_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
357
+ sql_lora_files):
358
lora_config = LoRAConfig(max_lora_rank=8, max_cpu_loras=4, max_loras=4)
359
- worker_lora_manager = LRUCacheWorkerLoRAManager(
359
+ worker_adapter_manager = LRUCacheWorkerLoRAManager(
360
4, 2, llama_2_7b_model_extra_embeddings.unpadded_vocab_size -
361
lora_config.lora_extra_vocab_size, lora_config, torch.device("cuda"),
362
EMBEDDING_MODULES, EMBEDDING_PADDING_MODULES)
363
- worker_lora_manager.create_lora_manager(llama_2_7b_model_extra_embeddings)
363
+ worker_adapter_manager.create_lora_manager(
364
+ llama_2_7b_model_extra_embeddings)
365
366
mapping = LoRAMapping([], [])
366
- worker_lora_manager.set_active_loras([
367
+ worker_adapter_manager.set_active_adapters([
368
LoRARequest("1", 1, sql_lora_files),
369
LoRARequest("2", 2, sql_lora_files)
370
], mapping)
370
- assert worker_lora_manager.list_loras() == {1, 2}
371
- assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
372
- assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
371
+ assert worker_adapter_manager.list_adapters() == {1, 2}
372
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[0] == 1
373
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[1] == 2
374
374
- worker_lora_manager.set_active_loras([
375
+ worker_adapter_manager.set_active_adapters([
376
LoRARequest("1", 1, sql_lora_files),
377
LoRARequest("3", 3, sql_lora_files),
378
LoRARequest("4", 4, sql_lora_files)
379
], mapping)
379
- assert worker_lora_manager.list_loras() == {1, 2, 3, 4}
380
- assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
381
- assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
382
- assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 3
383
- assert worker_lora_manager._lora_manager.lora_index_to_id[3] == 4
380
+ assert worker_adapter_manager.list_adapters() == {1, 2, 3, 4}
381
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[0] == 1
382
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[1] == 2
383
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[2] == 3
384
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[3] == 4
385
385
- worker_lora_manager.set_active_loras([
386
+ worker_adapter_manager.set_active_adapters([
387
LoRARequest("1", 1, sql_lora_files),
388
LoRARequest("2", 2, sql_lora_files),
389
LoRARequest("5", 5, sql_lora_files)
390
], mapping)
390
- assert worker_lora_manager.list_loras() == {1, 2, 4, 5}
391
- assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
392
- assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
393
- assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 5
394
- assert worker_lora_manager._lora_manager.lora_index_to_id[3] == 4
391
+ assert worker_adapter_manager.list_adapters() == {1, 2, 4, 5}
392
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[0] == 1
393
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[1] == 2
394
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[2] == 5
395
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[3] == 4
396
396
- worker_lora_manager.set_active_loras([
397
+ worker_adapter_manager.set_active_adapters([
398
LoRARequest("1", 1, sql_lora_files),
399
LoRARequest("1", 1, sql_lora_files),
400
LoRARequest("1", 1, sql_lora_files)
401
], mapping)
401
- assert worker_lora_manager.list_loras() == {1, 2, 4, 5}
402
- assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
403
- assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
404
- assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 5
405
- assert worker_lora_manager._lora_manager.lora_index_to_id[3] == 4
402
+ assert worker_adapter_manager.list_adapters() == {1, 2, 4, 5}
403
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[0] == 1
404
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[1] == 2
405
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[2] == 5
406
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[3] == 4
407
407
- worker_lora_manager.set_active_loras([
408
+ worker_adapter_manager.set_active_adapters([
409
LoRARequest("6", 6, sql_lora_files),
410
LoRARequest("7", 7, sql_lora_files),
411
LoRARequest("8", 8, sql_lora_files)
412
], mapping)
412
- assert worker_lora_manager.list_loras() == {1, 6, 7, 8}
413
- assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
414
- assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 7
415
- assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 8
416
- assert worker_lora_manager._lora_manager.lora_index_to_id[3] == 6
413
+ assert worker_adapter_manager.list_adapters() == {1, 6, 7, 8}
414
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[0] == 1
415
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[1] == 7
416
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[2] == 8
417
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[3] == 6
418
419
# Over capacity
420
with pytest.raises(RuntimeError):
420
- worker_lora_manager.set_active_loras([
421
+ worker_adapter_manager.set_active_adapters([
422
LoRARequest("10", 10, sql_lora_files),
423
LoRARequest("11", 11, sql_lora_files),
424
LoRARequest("12", 12, sql_lora_files),
@@ -426,68 +427,69 @@ def test_lru_cache_worker_lora_manager(llama_2_7b_model_extra_embeddings,
427
], mapping)
428
429
429
-def test_worker_lora_manager(llama_2_7b_model_extra_embeddings,
430
- sql_lora_files):
430
+def test_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
431
+ sql_lora_files):
432
# Should remove every LoRA not specified in the request.
433
lora_config = LoRAConfig(max_lora_rank=8, max_cpu_loras=4, max_loras=4)
433
- worker_lora_manager = WorkerLoRAManager(
434
+ worker_adapter_manager = WorkerLoRAManager(
435
4, 2, llama_2_7b_model_extra_embeddings.unpadded_vocab_size -
436
lora_config.lora_extra_vocab_size, lora_config, torch.device("cuda"),
437
EMBEDDING_MODULES, EMBEDDING_PADDING_MODULES)
437
- worker_lora_manager.create_lora_manager(llama_2_7b_model_extra_embeddings)
438
+ worker_adapter_manager.create_lora_manager(
439
+ llama_2_7b_model_extra_embeddings)
440
441
mapping = LoRAMapping([], [])
440
- worker_lora_manager.set_active_loras([
442
+ worker_adapter_manager.set_active_adapters([
443
LoRARequest("1", 1, sql_lora_files),
444
LoRARequest("2", 2, sql_lora_files)
445
], mapping)
444
- assert worker_lora_manager.list_loras() == {1, 2}
445
- assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
446
- assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
446
+ assert worker_adapter_manager.list_adapters() == {1, 2}
447
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[0] == 1
448
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[1] == 2
449
448
- worker_lora_manager.set_active_loras([
450
+ worker_adapter_manager.set_active_adapters([
451
LoRARequest("1", 1, sql_lora_files),
452
LoRARequest("3", 3, sql_lora_files),
453
LoRARequest("4", 4, sql_lora_files)
454
], mapping)
453
- assert worker_lora_manager.list_loras() == {1, 3, 4}
454
- assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
455
- assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 3
456
- assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 4
455
+ assert worker_adapter_manager.list_adapters() == {1, 3, 4}
456
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[0] == 1
457
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[1] == 3
458
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[2] == 4
459
458
- worker_lora_manager.set_active_loras([
460
+ worker_adapter_manager.set_active_adapters([
461
LoRARequest("1", 1, sql_lora_files),
462
LoRARequest("2", 2, sql_lora_files),
463
LoRARequest("5", 5, sql_lora_files)
464
], mapping)
463
- assert worker_lora_manager.list_loras() == {1, 2, 5}
464
- assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
465
- assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
466
- assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 5
465
+ assert worker_adapter_manager.list_adapters() == {1, 2, 5}
466
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[0] == 1
467
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[1] == 2
468
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[2] == 5
469
468
- worker_lora_manager.set_active_loras([
470
+ worker_adapter_manager.set_active_adapters([
471
LoRARequest("1", 1, sql_lora_files),
472
LoRARequest("1", 1, sql_lora_files),
473
LoRARequest("1", 1, sql_lora_files)
474
], mapping)
473
- assert worker_lora_manager.list_loras() == {1}
474
- assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
475
- assert worker_lora_manager._lora_manager.lora_index_to_id[1] is None
476
- assert worker_lora_manager._lora_manager.lora_index_to_id[2] is None
475
+ assert worker_adapter_manager.list_adapters() == {1}
476
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[0] == 1
477
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[1] is None
478
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[2] is None
479
478
- worker_lora_manager.set_active_loras([
480
+ worker_adapter_manager.set_active_adapters([
481
LoRARequest("6", 6, sql_lora_files),
482
LoRARequest("7", 7, sql_lora_files),
483
LoRARequest("8", 8, sql_lora_files)
484
], mapping)
483
- assert worker_lora_manager.list_loras() == {6, 7, 8}
484
- assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 8
485
- assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 6
486
- assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 7
485
+ assert worker_adapter_manager.list_adapters() == {6, 7, 8}
486
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[0] == 8
487
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[1] == 6
488
+ assert worker_adapter_manager._adapter_manager.lora_index_to_id[2] == 7
489
490
# Over capacity
491
with pytest.raises(RuntimeError):
490
- worker_lora_manager.set_active_loras([
492
+ worker_adapter_manager.set_active_adapters([
493
LoRARequest("10", 10, sql_lora_files),
494
LoRARequest("11", 11, sql_lora_files),
495
LoRARequest("12", 12, sql_lora_files),
@@ -525,8 +527,8 @@ def test_packed_loras(dist_init, dummy_model_gate_up):
527
528
assert isinstance(model.get_submodule("gate_up_proj"),
529
MergedColumnParallelLinearWithLoRA)
528
- assert manager.add_lora(model_lora)
529
- assert manager.add_lora(model_lora1)
530
+ assert manager.add_adapter(model_lora)
531
+ assert manager.add_adapter(model_lora1)
532
533
packed_lora = model_lora.get_lora("gate_up_proj")
534
assert packed_lora and isinstance(packed_lora, PackedLoRALayerWeights)
tests/prompt_adapter/test_bloom.py
new
+45
@@ -0,0 +1,45 @@
1
+import pytest
2
+
3
+import vllm
4
+from vllm.prompt_adapter.request import PromptAdapterRequest
5
+
6
+MODEL_PATH = "bigscience/bloomz-560m"
7
+PA_PATH = 'stevhliu/bloomz-560m_PROMPT_TUNING_CAUSAL_LM'
8
+
9
+
10
+def do_sample(llm, pa_name: str, pa_id: int):
11
+
12
+ prompts = [
13
+ "Tweet text : @nationalgridus I have no water and the bill is \
14
+ current and paid. Can you do something about this? Label : ",
15
+ "Tweet text : @nationalgridus Looks good thanks! Label : "
16
+ ]
17
+ sampling_params = vllm.SamplingParams(temperature=0.0,
18
+ max_tokens=3,
19
+ stop_token_ids=[3])
20
+
21
+ outputs = llm.generate(prompts,
22
+ sampling_params,
23
+ prompt_adapter_request=PromptAdapterRequest(
24
+ pa_name, pa_id, PA_PATH, 8) if pa_id else None)
25
+
26
+ # Print the outputs.
27
+ generated_texts = []
28
+ for output in outputs:
29
+ prompt = output.prompt
30
+ generated_text = output.outputs[0].text.strip()
31
+ generated_texts.append(generated_text)
32
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
33
+ return generated_texts
34
+
35
+
36
+@pytest.mark.parametrize("enforce_eager", [True, False])
37
+def test_twitter_prompt_adapter(enforce_eager: bool):
38
+ llm = vllm.LLM(MODEL_PATH,
39
+ enforce_eager=enforce_eager,
40
+ enable_prompt_adapter=True,
41
+ max_prompt_adapter_token=8)
42
+
43
+ expected_output = ['complaint', 'no complaint']
44
+
45
+ assert do_sample(llm, "twitter_pa", pa_id=1) == expected_output
tests/prompt_adapter/test_multi_adapter_inference.py
new
+53
@@ -0,0 +1,53 @@
1
+from vllm import EngineArgs, LLMEngine, SamplingParams
2
+from vllm.prompt_adapter.request import PromptAdapterRequest
3
+
4
+MODEL_PATH = "bigscience/bloomz-560m"
5
+pa_path = 'stevhliu/bloomz-560m_PROMPT_TUNING_CAUSAL_LM'
6
+pa_path2 = 'swapnilbp/angry_tweet_ptune'
7
+
8
+
9
+def do_sample(engine):
10
+
11
+ prompts = [
12
+ ("Tweet text: I have complaints! Label: ",
13
+ SamplingParams(temperature=0.0, max_tokens=3, stop_token_ids=[3]),
14
+ PromptAdapterRequest("hate_speech", 1, pa_path2, 8)),
15
+ ("Tweet text: I have no problems Label: ",
16
+ SamplingParams(temperature=0.0, max_tokens=3, stop_token_ids=[3]),
17
+ PromptAdapterRequest("hate_speech2", 2, pa_path2, 8)),
18
+ ("Tweet text: I have complaints! Label: ",
19
+ SamplingParams(temperature=0.0, max_tokens=3), None),
20
+ ("Tweet text: I have no problems Label: ",
21
+ SamplingParams(temperature=0.0, max_tokens=3, stop_token_ids=[3]),
22
+ PromptAdapterRequest("complain", 3, pa_path, 8)),
23
+ ]
24
+
25
+ request_id = 0
26
+ results = set()
27
+ while prompts or engine.has_unfinished_requests():
28
+ if prompts:
29
+ prompt, sampling_params, pa_request = prompts.pop(0)
30
+ engine.add_request(str(request_id),
31
+ prompt,
32
+ sampling_params,
33
+ prompt_adapter_request=pa_request)
34
+ request_id += 1
35
+
36
+ request_outputs = engine.step()
37
+
38
+ for request_output in request_outputs:
39
+ if request_output.finished:
40
+ results.add(request_output.outputs[0].text)
41
+ return results
42
+
43
+
44
+def test_multi_prompt_adapters():
45
+ engine_args = EngineArgs(model=MODEL_PATH,
46
+ max_prompt_adapters=3,
47
+ enable_prompt_adapter=True,
48
+ max_prompt_adapter_token=8)
49
+ engine = LLMEngine.from_engine_args(engine_args)
50
+ expected_output = {
51
+ ' quot;I', 'hate speech', 'no complaint', 'not hate speech'
52
+ }
53
+ assert do_sample(engine) == expected_output
tests/prompt_adapter/test_pa_lora.py
new
+61
@@ -0,0 +1,61 @@
1
+from huggingface_hub import snapshot_download
2
+
3
+from vllm import EngineArgs, LLMEngine, SamplingParams
4
+from vllm.lora.request import LoRARequest
5
+from vllm.prompt_adapter.request import PromptAdapterRequest
6
+
7
+MODEL_PATH = "meta-llama/Llama-2-7b-hf"
8
+pa_path = snapshot_download(repo_id="swapnilbp/llama_tweet_ptune")
9
+lora_path = snapshot_download(repo_id="yard1/llama-2-7b-sql-lora-test")
10
+
11
+
12
+def do_sample(engine):
13
+
14
+ prompt_text = "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]" # noqa: E501
15
+
16
+ # first prompt with a prompt adapter and second without adapter
17
+ prompts = [
18
+ (prompt_text,
19
+ SamplingParams(temperature=0.0, max_tokens=100,
20
+ stop=["[/assistant]"]),
21
+ PromptAdapterRequest("hate_speech", 1, pa_path,
22
+ 8), LoRARequest("sql_test", 1, lora_path)),
23
+ (prompt_text,
24
+ SamplingParams(temperature=0.0, max_tokens=100,
25
+ stop=["[/assistant]"]), None,
26
+ LoRARequest("sql_test", 1, lora_path)),
27
+ ]
28
+
29
+ request_id = 0
30
+ results = set()
31
+ while prompts or engine.has_unfinished_requests():
32
+ if prompts:
33
+ prompt, sampling_params, pa_request, lora_request = prompts.pop(0)
34
+ engine.add_request(str(request_id),
35
+ prompt,
36
+ sampling_params,
37
+ prompt_adapter_request=pa_request,
38
+ lora_request=lora_request)
39
+ request_id += 1
40
+
41
+ request_outputs = engine.step()
42
+
43
+ for request_output in request_outputs:
44
+ if request_output.finished:
45
+ results.add(request_output.outputs[0].text)
46
+ return results
47
+
48
+
49
+def test_lora_prompt_adapter():
50
+ engine_args = EngineArgs(model=MODEL_PATH,
51
+ enable_prompt_adapter=True,
52
+ enable_lora=True,
53
+ max_num_seqs=60,
54
+ max_prompt_adapter_token=8)
55
+ engine = LLMEngine.from_engine_args(engine_args)
56
+ result = do_sample(engine)
57
+
58
+ expected_output = {
59
+ " SELECT icao FROM table_name_74 WHERE airport = 'lilongwe international airport' " # noqa: E501
60
+ }
61
+ assert result == expected_output
tests/spec_decode/e2e/conftest.py
+2
@@ -13,6 +13,7 @@ from vllm.lora.request import LoRARequest
13
from vllm.model_executor.utils import set_random_seed
14
from vllm.multimodal import MultiModalDataDict
15
from vllm.outputs import RequestOutput
16
+from vllm.prompt_adapter.request import PromptAdapterRequest
17
from vllm.sampling_params import SamplingParams
18
from vllm.sequence import Logprob
19
from vllm.usage.usage_lib import UsageContext
@@ -92,6 +93,7 @@ class AsyncLLM:
93
use_tqdm: bool = True,
94
lora_request: Optional[LoRARequest] = None,
95
multi_modal_data: Optional[MultiModalDataDict] = None,
96
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None
97
) -> List[RequestOutput]:
98
99
if prompts is None:
tests/worker/test_model_runner.py
+1
@@ -23,6 +23,7 @@ def _create_model_runner(model: str, *args, **kwargs) -> ModelRunner:
23
cache_config=engine_config.cache_config,
24
load_config=engine_config.load_config,
25
lora_config=engine_config.lora_config,
26
+ prompt_adapter_config=engine_config.prompt_adapter_config,
27
is_driver_worker=True,
28
)
29
return model_runner
vllm/adapter_commons/__init__.py
vllm/adapter_commons/layers.py
new
+14
@@ -0,0 +1,14 @@
1
+from dataclasses import dataclass
2
+from typing import Tuple
3
+
4
+
5
+@dataclass
6
+class AdapterMapping:
7
+ # Per every token in input_ids:
8
+ index_mapping: Tuple[int, ...]
9
+ # Per sampled token:
10
+ prompt_mapping: Tuple[int, ...]
11
+
12
+ def __post_init__(self):
13
+ self.index_mapping = tuple(self.index_mapping)
14
+ self.prompt_mapping = tuple(self.prompt_mapping)
\ No newline at end of file
vllm/adapter_commons/models.py
new
+104
@@ -0,0 +1,104 @@
1
+from abc import ABC, abstractmethod
2
+from typing import Any, Callable, Dict, Hashable, Optional, TypeVar
3
+
4
+from torch import nn
5
+
6
+from vllm.logger import init_logger
7
+from vllm.utils import LRUCache
8
+
9
+logger = init_logger(__name__)
10
+
11
+
12
+class AdapterModel(ABC):
13
+
14
+ def __init__(self, model_id=None):
15
+ self.id = model_id
16
+
17
+ @abstractmethod
18
+ def from_local_checkpoint(cls, model_dir, model_id=None, **kwargs):
19
+ # Common initialization code
20
+ # Load weights or embeddings from local checkpoint
21
+ raise NotImplementedError("Subclasses must implement this method.")
22
+
23
+
24
+T = TypeVar('T')
25
+
26
+
27
+class AdapterLRUCache(LRUCache[T]):
28
+
29
+ def __init__(self, capacity: int, deactivate_fn: Callable[[Hashable],
30
+ None]):
31
+ super().__init__(capacity)
32
+ self.deactivate_fn = deactivate_fn
33
+
34
+ def _on_remove(self, key: Hashable, value: T):
35
+ logger.debug("Removing adapter int id: %d", key)
36
+ self.deactivate_fn(key)
37
+ return super()._on_remove(key, value)
38
+
39
+
40
+class AdapterModelManager(ABC):
41
+
42
+ def __init__(
43
+ self,
44
+ model: nn.Module,
45
+ ):
46
+ """Create a AdapterModelManager and adapter for a given model.
47
+ Args:
48
+ model: the model to be adapted.
49
+ """
50
+ self.model: nn.Module = model
51
+ self._registered_adapters: Dict[int, Any] = {}
52
+ # Dict instead of a Set for compatibility with LRUCache.
53
+ self._active_adapters: Dict[int, None] = {}
54
+ self.adapter_type = 'Adapter'
55
+ self._last_mapping = None
56
+
57
+ def __len__(self) -> int:
58
+ return len(self._registered_adapters)
59
+
60
+ @property
61
+ @abstractmethod
62
+ def adapter_slots(self):
63
+ ...
64
+
65
+ @property
66
+ @abstractmethod
67
+ def capacity(self):
68
+ ...
69
+
70
+ @abstractmethod
71
+ def activate_adapter(self, adapter_id: int) -> bool:
72
+ ...
73
+
74
+ @abstractmethod
75
+ def deactivate_adapter(self, adapter_id: int) -> bool:
76
+ ...
77
+
78
+ @abstractmethod
79
+ def add_adapter(self, adapter: Any) -> bool:
80
+ ...
81
+
82
+ @abstractmethod
83
+ def set_adapter_mapping(self, mapping: Any) -> None:
84
+ ...
85
+
86
+ @abstractmethod
87
+ def remove_adapter(self, adapter_id: int) -> bool:
88
+ ...
89
+
90
+ @abstractmethod
91
+ def remove_all_adapters(self):
92
+ ...
93
+
94
+ @abstractmethod
95
+ def get_adapter(self, adapter_id: int) -> Optional[Any]:
96
+ ...
97
+
98
+ @abstractmethod
99
+ def list_adapters(self) -> Dict[int, Any]:
100
+ ...
101
+
102
+ @abstractmethod
103
+ def pin_adapter(self, adapter_id: int) -> bool:
104
+ ...
vllm/adapter_commons/request.py
new
+25
@@ -0,0 +1,25 @@
1
+from abc import abstractmethod
2
+from dataclasses import dataclass
3
+
4
+
5
+@dataclass
6
+class AdapterRequest:
7
+ """
8
+ Base class for adapter requests.
9
+ """
10
+
11
+ @property
12
+ @abstractmethod
13
+ def adapter_id(self):
14
+ ...
15
+
16
+ def __post_init__(self):
17
+ if self.adapter_id < 1:
18
+ raise ValueError(f"id must be > 0, got {self.adapter_id}")
19
+
20
+ def __eq__(self, value: object) -> bool:
21
+ return isinstance(
22
+ value, self.__class__) and self.adapter_id == value.adapter_id
23
+
24
+ def __hash__(self) -> int:
25
+ return hash(self.adapter_id)
vllm/adapter_commons/utils.py
new
+90
@@ -0,0 +1,90 @@
1
+from typing import Any, Callable, Dict, Optional, Set
2
+
3
+
4
+## model functions
5
+def deactivate_adapter(adapter_id: int, active_adapters: Dict[int, None],
6
+ deactivate_func: Callable) -> bool:
7
+ if adapter_id in active_adapters:
8
+ deactivate_func(adapter_id)
9
+ active_adapters.pop(adapter_id)
10
+ return True
11
+ return False
12
+
13
+
14
+def add_adapter(adapter: Any, registered_adapters: Dict[int, Any],
15
+ capacity: int, add_func: Callable) -> bool:
16
+ if adapter.id not in registered_adapters:
17
+ if len(registered_adapters) >= capacity:
18
+ raise RuntimeError('No free adapter slots.')
19
+ add_func(adapter)
20
+ registered_adapters[adapter.id] = adapter
21
+ return True
22
+ return False
23
+
24
+
25
+def set_adapter_mapping(mapping: Any, last_mapping: Any,
26
+ set_mapping_func: Callable) -> Any:
27
+ if last_mapping != mapping:
28
+ set_mapping_func(mapping)
29
+ return mapping
30
+ return last_mapping
31
+
32
+
33
+def remove_adapter(adapter_id: int, registered_adapters: Dict[int, Any],
34
+ deactivate_func: Callable) -> bool:
35
+ deactivate_func(adapter_id)
36
+ return bool(registered_adapters.pop(adapter_id, None))
37
+
38
+
39
+def list_adapters(registered_adapters: Dict[int, Any]) -> Dict[int, Any]:
40
+ return dict(registered_adapters)
41
+
42
+
43
+def get_adapter(adapter_id: int,
44
+ registered_adapters: Dict[int, Any]) -> Optional[Any]:
45
+ return registered_adapters.get(adapter_id, None)
46
+
47
+
48
+## worker functions
49
+def set_active_adapters_worker(requests: Set[Any], mapping: Optional[Any],
50
+ apply_adapters_func,
51
+ set_adapter_mapping_func) -> None:
52
+ apply_adapters_func(requests)
53
+ set_adapter_mapping_func(mapping)
54
+
55
+
56
+def add_adapter_worker(adapter_request: Any, list_adapters_func,
57
+ load_adapter_func, add_adapter_func,
58
+ activate_adapter_func) -> bool:
59
+ if adapter_request.adapter_id in list_adapters_func():
60
+ return False
61
+ loaded_adapter = load_adapter_func(adapter_request)
62
+ loaded = add_adapter_func(loaded_adapter)
63
+ activate_adapter_func(loaded_adapter.id)
64
+ return loaded
65
+
66
+
67
+def apply_adapters_worker(adapter_requests: Set[Any], list_adapters_func,
68
+ adapter_slots: int, remove_adapter_func,
69
+ add_adapter_func) -> None:
70
+ models_that_exist = list_adapters_func()
71
+ models_map = {
72
+ adapter_request.adapter_id: adapter_request
73
+ for adapter_request in adapter_requests if adapter_request
74
+ }
75
+ if len(models_map) > adapter_slots:
76
+ raise RuntimeError(
77
+ f"Number of requested models ({len(models_map)}) is greater "
78
+ f"than the number of GPU model slots "
79
+ f"({adapter_slots}).")
80
+ new_models = set(models_map)
81
+ models_to_add = new_models - models_that_exist
82
+ models_to_remove = models_that_exist - new_models
83
+ for adapter_id in models_to_remove:
84
+ remove_adapter_func(adapter_id)
85
+ for adapter_id in models_to_add:
86
+ add_adapter_func(models_map[adapter_id])
87
+
88
+
89
+def list_adapters_worker(adapter_manager_list_adapters_func) -> Set[int]:
90
+ return set(adapter_manager_list_adapters_func())
vllm/adapter_commons/worker_manager.py
new
+36
@@ -0,0 +1,36 @@
1
+from abc import ABC, abstractmethod
2
+from typing import Any, Optional, Set
3
+
4
+import torch
5
+
6
+
7
+class AbstractWorkerManager(ABC):
8
+
9
+ def __init__(self, device: torch.device):
10
+ self.device = device
11
+
12
+ @property
13
+ @abstractmethod
14
+ def is_enabled(self) -> bool:
15
+ ...
16
+
17
+ @abstractmethod
18
+ def set_active_adapters(self, requests: Set[Any],
19
+ mapping: Optional[Any]) -> None:
20
+ ...
21
+
22
+ @abstractmethod
23
+ def add_adapter(self, adapter_request: Any) -> bool:
24
+ ...
25
+
26
+ @abstractmethod
27
+ def remove_adapter(self, adapter_id: int) -> bool:
28
+ ...
29
+
30
+ @abstractmethod
31
+ def remove_all_adapters(self):
32
+ ...
33
+
34
+ @abstractmethod
35
+ def list_adapters(self) -> Set[int]:
36
+ ...
vllm/config.py
+37
@@ -1285,6 +1285,39 @@ class LoRAConfig:
1285
raise ValueError("LoRA is not supported with chunked prefill yet.")
1286
1287
1288
+@dataclass
1289
+class PromptAdapterConfig:
1290
+ max_prompt_adapters: int
1291
+ max_prompt_adapter_token: int
1292
+ max_cpu_prompt_adapters: Optional[int] = None
1293
+ prompt_adapter_dtype: Optional[torch.dtype] = None
1294
+
1295
+ def __post_init__(self):
1296
+ library_name = 'peft'
1297
+ try:
1298
+ __import__(library_name)
1299
+ except ImportError as e:
1300
+ raise ImportError(
1301
+ f"'{library_name}' is not installed for prompt adapter support."
1302
+ f"Please install it using 'pip install {library_name}'."
1303
+ ) from e
1304
+
1305
+ if self.max_prompt_adapters < 1:
1306
+ raise ValueError(f"max_prompt_adapters "
1307
+ f"({self.max_prompt_adapters}) must be >= 1.")
1308
+ if self.max_prompt_adapter_token == 0:
1309
+ raise ValueError("max_prompt_adapter_token must be set.")
1310
+ if self.max_cpu_prompt_adapters is None:
1311
+ self.max_cpu_prompt_adapters = self.max_prompt_adapters
1312
+
1313
+ def verify_with_model_config(self, model_config: ModelConfig):
1314
+ if self.prompt_adapter_dtype in (None, "auto"):
1315
+ self.prompt_adapter_dtype = model_config.dtype
1316
+ elif isinstance(self.prompt_adapter_dtype, str):
1317
+ self.prompt_adapter_dtype = getattr(torch,
1318
+ self.prompt_adapter_dtype)
1319
+
1320
+
1321
@dataclass
1322
class MultiModalConfig:
1323
"""Configs the input data format and how models should run for
@@ -1518,6 +1551,7 @@ class EngineConfig:
1551
speculative_config: Optional[SpeculativeConfig]
1552
decoding_config: Optional[DecodingConfig]
1553
observability_config: Optional[ObservabilityConfig]
1554
+ prompt_adapter_config: Optional[PromptAdapterConfig]
1555
1556
def __post_init__(self):
1557
"""Verify configs are valid & consistent with each other.
@@ -1529,6 +1563,9 @@ class EngineConfig:
1563
self.lora_config.verify_with_model_config(self.model_config)
1564
self.lora_config.verify_with_scheduler_config(
1565
self.scheduler_config)
1566
+ if self.prompt_adapter_config:
1567
+ self.prompt_adapter_config.verify_with_model_config(
1568
+ self.model_config)
1569
1570
def to_dict(self):
1571
"""Return the configs as a dictionary, for use in **kwargs.
vllm/core/scheduler.py
+12
@@ -11,6 +11,7 @@ from vllm.core.interfaces import AllocStatus, BlockSpaceManager
11
from vllm.core.policy import Policy, PolicyFactory
12
from vllm.logger import init_logger
13
from vllm.lora.request import LoRARequest
14
+from vllm.prompt_adapter.request import PromptAdapterRequest
15
from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
16
SequenceGroupMetadata, SequenceStatus)
17
@@ -139,6 +140,8 @@ class SchedulerOutputs:
140
if self.num_loras > 0:
141
self._sort_by_lora_ids()
142
143
+ self.num_prompt_adapters: int = len(self.prompt_adapter_requests)
144
+
145
def is_empty(self) -> bool:
146
# NOTE: We do not consider the ignored sequence groups.
147
return (not self.scheduled_seq_groups and not self.blocks_to_swap_in
@@ -157,6 +160,14 @@ class SchedulerOutputs:
160
if g.seq_group.lora_request is not None
161
}
162
163
+ @property
164
+ def prompt_adapter_requests(self) -> Set[PromptAdapterRequest]:
165
+ return {
166
+ g.seq_group.prompt_adapter_request
167
+ for g in self.scheduled_seq_groups
168
+ if g.seq_group.prompt_adapter_request is not None
169
+ }
170
+
171
172
@dataclass
173
class SchedulerRunningOutputs:
@@ -1024,6 +1035,7 @@ class Scheduler:
1035
# `multi_modal_data` will be None.
1036
multi_modal_data=seq_group.multi_modal_data
1037
if scheduler_outputs.num_prefill_groups > 0 else None,
1038
+ prompt_adapter_request=seq_group.prompt_adapter_request,
1039
)
1040
seq_group_metadata_list.append(seq_group_metadata)
1041
vllm/engine/arg_utils.py
+22
-2
@@ -7,8 +7,8 @@ from typing import List, Optional, Tuple, Union
7
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
8
EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
9
MultiModalConfig, ObservabilityConfig, ParallelConfig,
10
- SchedulerConfig, SpeculativeConfig,
11
- TokenizerPoolConfig)
10
+ PromptAdapterConfig, SchedulerConfig,
11
+ SpeculativeConfig, TokenizerPoolConfig)
12
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
13
from vllm.utils import FlexibleArgumentParser
14
@@ -66,6 +66,9 @@ class EngineArgs:
66
enable_lora: bool = False
67
max_loras: int = 1
68
max_lora_rank: int = 16
69
+ enable_prompt_adapter: bool = False
70
+ max_prompt_adapters: int = 1
71
+ max_prompt_adapter_token: int = 0
72
fully_sharded_loras: bool = False
73
lora_extra_vocab_size: int = 256
74
long_lora_scaling_factors: Optional[Tuple[float]] = None
@@ -449,6 +452,17 @@ class EngineArgs:
452
'Enabling this will use the fully sharded layers. '
453
'At high sequence length, max rank or '
454
'tensor parallel size, this is likely faster.'))
455
+ parser.add_argument('--enable-prompt-adapter',
456
+ action='store_true',
457
+ help='If True, enable handling of PromptAdapters.')
458
+ parser.add_argument('--max-prompt-adapters',
459
+ type=int,
460
+ default=EngineArgs.max_prompt_adapters,
461
+ help='Max number of PromptAdapters in a batch.')
462
+ parser.add_argument('--max-prompt-adapter-token',
463
+ type=int,
464
+ default=EngineArgs.max_prompt_adapter_token,
465
+ help='Max number of PromptAdapters tokens')
466
parser.add_argument("--device",
467
type=str,
468
default=EngineArgs.device,
@@ -726,6 +740,11 @@ class EngineArgs:
740
model_loader_extra_config=self.model_loader_extra_config,
741
)
742
743
+ prompt_adapter_config = PromptAdapterConfig(
744
+ max_prompt_adapters=self.max_prompt_adapters,
745
+ max_prompt_adapter_token=self.max_prompt_adapter_token) \
746
+ if self.enable_prompt_adapter else None
747
+
748
decoding_config = DecodingConfig(
749
guided_decoding_backend=self.guided_decoding_backend)
750
@@ -751,6 +770,7 @@ class EngineArgs:
770
load_config=load_config,
771
decoding_config=decoding_config,
772
observability_config=observability_config,
773
+ prompt_adapter_config=prompt_adapter_config,
774
)
775
776
vllm/engine/async_llm_engine.py
+29
-9
@@ -18,6 +18,7 @@ from vllm.logger import init_logger
18
from vllm.lora.request import LoRARequest
19
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
20
from vllm.pooling_params import PoolingParams
21
+from vllm.prompt_adapter.request import PromptAdapterRequest
22
from vllm.sampling_params import SamplingParams
23
from vllm.sequence import ExecuteModelRequest, SamplerOutput
24
from vllm.usage.usage_lib import UsageContext
@@ -264,6 +265,7 @@ class _AsyncLLMEngine(LLMEngine):
265
request_id: str,
266
inputs: PromptInputs,
267
lora_request: Optional[LoRARequest] = None,
268
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
269
) -> LLMInputs:
270
if isinstance(inputs, str):
271
inputs = {"prompt": inputs}
@@ -279,6 +281,12 @@ class _AsyncLLMEngine(LLMEngine):
281
else:
282
prompt_token_ids = inputs["prompt_token_ids"]
283
284
+ if prompt_adapter_request:
285
+ prompt_token_ids = [
286
+ 0
287
+ ] * prompt_adapter_request.prompt_adapter_num_virtual_tokens + \
288
+ prompt_token_ids
289
+
290
llm_inputs = LLMInputs(prompt_token_ids=prompt_token_ids,
291
prompt=inputs.get("prompt"),
292
multi_modal_data=inputs.get("multi_modal_data"))
@@ -286,13 +294,14 @@ class _AsyncLLMEngine(LLMEngine):
294
return self.input_processor(llm_inputs)
295
296
async def add_request_async(
289
- self,
290
- request_id: str,
291
- inputs: PromptInputs,
292
- params: Union[SamplingParams, PoolingParams],
293
- arrival_time: Optional[float] = None,
294
- lora_request: Optional[LoRARequest] = None,
295
- trace_headers: Optional[Dict[str, str]] = None,
297
+ self,
298
+ request_id: str,
299
+ inputs: PromptInputs,
300
+ params: Union[SamplingParams, PoolingParams],
301
+ arrival_time: Optional[float] = None,
302
+ lora_request: Optional[LoRARequest] = None,
303
+ trace_headers: Optional[Dict[str, str]] = None,
304
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None
305
) -> None:
306
if lora_request is not None and not self.lora_config:
307
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
@@ -301,7 +310,10 @@ class _AsyncLLMEngine(LLMEngine):
310
arrival_time = time.time()
311
312
processed_inputs = await self.process_model_inputs_async(
304
- request_id=request_id, inputs=inputs, lora_request=lora_request)
313
+ request_id=request_id,
314
+ inputs=inputs,
315
+ lora_request=lora_request,
316
+ prompt_adapter_request=prompt_adapter_request)
317
318
self._add_processed_request(
319
request_id=request_id,
@@ -309,6 +321,7 @@ class _AsyncLLMEngine(LLMEngine):
321
params=params,
322
arrival_time=arrival_time,
323
lora_request=lora_request,
324
+ prompt_adapter_request=prompt_adapter_request,
325
trace_headers=trace_headers,
326
)
327
@@ -627,6 +640,7 @@ class AsyncLLMEngine:
640
arrival_time: Optional[float] = None,
641
lora_request: Optional[LoRARequest] = None,
642
trace_headers: Optional[Dict[str, str]] = None,
643
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None
644
) -> AsyncStream:
645
if self.log_requests:
646
if isinstance(inputs, str):
@@ -669,7 +683,7 @@ class AsyncLLMEngine:
683
arrival_time=arrival_time,
684
lora_request=lora_request,
685
trace_headers=trace_headers,
672
- )
686
+ prompt_adapter_request=prompt_adapter_request)
687
688
return stream
689
@@ -680,6 +694,7 @@ class AsyncLLMEngine:
694
request_id: str,
695
lora_request: Optional[LoRARequest] = None,
696
trace_headers: Optional[Dict[str, str]] = None,
697
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None
698
) -> AsyncIterator[RequestOutput]:
699
"""Generate outputs for a request.
700
@@ -695,6 +710,8 @@ class AsyncLLMEngine:
710
request_id: The unique id of the request.
711
lora_request: LoRA request to use for generation, if any.
712
trace_headers: OpenTelemetry trace headers.
713
+ prompt_adapter_request: Prompt Adapter request to use
714
+ for generation, if any.
715
716
Yields:
717
The output `RequestOutput` objects from the LLMEngine
@@ -749,6 +766,7 @@ class AsyncLLMEngine:
766
sampling_params,
767
lora_request=lora_request,
768
trace_headers=trace_headers,
769
+ prompt_adapter_request=prompt_adapter_request,
770
):
771
yield LLMEngine.validate_output(output, RequestOutput)
772
@@ -837,6 +855,7 @@ class AsyncLLMEngine:
855
*,
856
lora_request: Optional[LoRARequest] = None,
857
trace_headers: Optional[Dict[str, str]] = None,
858
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
859
) -> AsyncIterator[Union[RequestOutput, EmbeddingRequestOutput]]:
860
"""Common logic to process requests with SamplingParams or
861
PoolingParams."""
@@ -849,6 +868,7 @@ class AsyncLLMEngine:
868
arrival_time=arrival_time,
869
lora_request=lora_request,
870
trace_headers=trace_headers,
871
+ prompt_adapter_request=prompt_adapter_request,
872
)
873
874
try:
vllm/engine/llm_engine.py
+50
-15
@@ -8,7 +8,8 @@ from transformers import PreTrainedTokenizer
8
9
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
10
LoRAConfig, ModelConfig, MultiModalConfig,
11
- ObservabilityConfig, ParallelConfig, SchedulerConfig,
11
+ ObservabilityConfig, ParallelConfig,
12
+ PromptAdapterConfig, SchedulerConfig,
13
SpeculativeConfig)
14
from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler,
15
SchedulerOutputs)
@@ -27,6 +28,7 @@ from vllm.lora.request import LoRARequest
28
from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
29
RequestOutputFactory)
30
from vllm.pooling_params import PoolingParams
31
+from vllm.prompt_adapter.request import PromptAdapterRequest
32
from vllm.sampling_params import SamplingParams
33
from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
34
PoolerOutput, SamplerOutput, Sequence,
@@ -93,6 +95,8 @@ class LLMEngine:
95
decoding.
96
executor_class: The model executor class for managing distributed
97
execution.
98
+ prompt_adapter_config (Optional): The configuration related to serving
99
+ prompt adapters.
100
log_stats: Whether to log statistics.
101
usage_context: Specified entry point, used for usage info collection.
102
"""
@@ -161,6 +165,7 @@ class LLMEngine:
165
speculative_config: Optional[SpeculativeConfig],
166
decoding_config: Optional[DecodingConfig],
167
observability_config: Optional[ObservabilityConfig],
168
+ prompt_adapter_config: Optional[PromptAdapterConfig],
169
executor_class: Type[ExecutorBase],
170
log_stats: bool,
171
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
@@ -222,6 +227,7 @@ class LLMEngine:
227
self.speculative_config = speculative_config
228
self.load_config = load_config
229
self.decoding_config = decoding_config or DecodingConfig()
230
+ self.prompt_adapter_config = prompt_adapter_config
231
self.observability_config = observability_config or ObservabilityConfig(
232
)
233
self.log_stats = log_stats
@@ -250,6 +256,7 @@ class LLMEngine:
256
multimodal_config=multimodal_config,
257
speculative_config=speculative_config,
258
load_config=load_config,
259
+ prompt_adapter_config=prompt_adapter_config,
260
)
261
262
if not self.model_config.embedding_mode:
@@ -282,6 +289,8 @@ class LLMEngine:
289
# Feature flags
290
"enable_lora":
291
bool(lora_config),
292
+ "enable_prompt_adapter":
293
+ bool(prompt_adapter_config),
294
"enable_prefix_caching":
295
cache_config.enable_prefix_caching,
296
"enforce_eager":
@@ -376,7 +385,6 @@ class LLMEngine:
385
engine_config = engine_args.create_engine_config()
386
distributed_executor_backend = (
387
engine_config.parallel_config.distributed_executor_backend)
379
-
388
# Initialize the cluster and specify the executor class.
389
if engine_config.device_config.device_type == "neuron":
390
from vllm.executor.neuron_executor import NeuronExecutor
@@ -409,7 +417,6 @@ class LLMEngine:
417
else:
418
from vllm.executor.gpu_executor import GPUExecutor
419
executor_class = GPUExecutor
412
-
420
# Create the LLM engine.
421
engine = cls(
422
**engine_config.to_dict(),
@@ -470,6 +477,9 @@ class LLMEngine:
477
self.lora_config.verify_with_model_config(self.model_config)
478
self.lora_config.verify_with_scheduler_config(
479
self.scheduler_config)
480
+ if self.prompt_adapter_config:
481
+ self.prompt_adapter_config.verify_with_model_config(
482
+ self.model_config)
483
484
def _get_eos_token_id(
485
self, lora_request: Optional[LoRARequest]) -> Optional[int]:
@@ -487,6 +497,7 @@ class LLMEngine:
497
params: Union[SamplingParams, PoolingParams],
498
arrival_time: float,
499
lora_request: Optional[LoRARequest],
500
+ prompt_adapter_request: Optional[PromptAdapterRequest],
501
trace_headers: Optional[Dict[str, str]] = None,
502
) -> None:
503
# Create the sequences.
@@ -495,7 +506,7 @@ class LLMEngine:
506
eos_token_id = self._get_eos_token_id(lora_request)
507
508
seq = Sequence(seq_id, processed_inputs, block_size, eos_token_id,
498
- lora_request)
509
+ lora_request, prompt_adapter_request)
510
511
# Create a SequenceGroup based on SamplingParams or PoolingParams
512
if isinstance(params, SamplingParams):
@@ -506,7 +517,7 @@ class LLMEngine:
517
arrival_time=arrival_time,
518
lora_request=lora_request,
519
trace_headers=trace_headers,
509
- )
520
+ prompt_adapter_request=prompt_adapter_request)
521
elif isinstance(params, PoolingParams):
522
seq_group = self._create_sequence_group_with_pooling(
523
request_id,
@@ -514,7 +525,7 @@ class LLMEngine:
525
params,
526
arrival_time=arrival_time,
527
lora_request=lora_request,
517
- )
528
+ prompt_adapter_request=prompt_adapter_request)
529
else:
530
raise ValueError(
531
"Either SamplingParams or PoolingParams must be provided.")
@@ -535,6 +546,7 @@ class LLMEngine:
546
request_id: str,
547
inputs: PromptInputs,
548
lora_request: Optional[LoRARequest] = None,
549
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
550
) -> LLMInputs:
551
if isinstance(inputs, str):
552
inputs = {"prompt": inputs}
@@ -549,6 +561,11 @@ class LLMEngine:
561
else:
562
prompt_token_ids = inputs["prompt_token_ids"]
563
564
+ if prompt_adapter_request:
565
+ prompt_token_ids = \
566
+ [0] * prompt_adapter_request.prompt_adapter_num_virtual_tokens\
567
+ + prompt_token_ids
568
+
569
llm_inputs = LLMInputs(prompt_token_ids=prompt_token_ids,
570
prompt=inputs.get("prompt"),
571
multi_modal_data=inputs.get("multi_modal_data"))
@@ -563,6 +580,7 @@ class LLMEngine:
580
arrival_time: Optional[float] = None,
581
lora_request: Optional[LoRARequest] = None,
582
trace_headers: Optional[Dict[str, str]] = None,
583
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
584
) -> None:
585
"""Add a request to the engine's request pool.
586
@@ -612,9 +630,11 @@ class LLMEngine:
630
if arrival_time is None:
631
arrival_time = time.time()
632
615
- processed_inputs = self.process_model_inputs(request_id=request_id,
616
- inputs=inputs,
617
- lora_request=lora_request)
633
+ processed_inputs = self.process_model_inputs(
634
+ request_id=request_id,
635
+ inputs=inputs,
636
+ lora_request=lora_request,
637
+ prompt_adapter_request=prompt_adapter_request)
638
639
self._add_processed_request(
640
request_id=request_id,
@@ -622,6 +642,7 @@ class LLMEngine:
642
params=params,
643
arrival_time=arrival_time,
644
lora_request=lora_request,
645
+ prompt_adapter_request=prompt_adapter_request,
646
trace_headers=trace_headers,
647
)
648
@@ -633,6 +654,7 @@ class LLMEngine:
654
arrival_time: float,
655
lora_request: Optional[LoRARequest],
656
trace_headers: Optional[Dict[str, str]] = None,
657
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
658
) -> SequenceGroup:
659
"""Creates a SequenceGroup with SamplingParams."""
660
max_logprobs = self.get_model_config().max_logprobs
@@ -658,7 +680,7 @@ class LLMEngine:
680
sampling_params=sampling_params,
681
lora_request=lora_request,
682
trace_headers=trace_headers,
661
- )
683
+ prompt_adapter_request=prompt_adapter_request)
684
685
return seq_group
686
@@ -669,16 +691,19 @@ class LLMEngine:
691
pooling_params: PoolingParams,
692
arrival_time: float,
693
lora_request: Optional[LoRARequest],
694
+ prompt_adapter_request: Optional[PromptAdapterRequest],
695
) -> SequenceGroup:
696
"""Creates a SequenceGroup with PoolingParams."""
697
# Defensive copy of PoolingParams, which are used by the pooler
698
pooling_params = pooling_params.clone()
699
# Create the sequence group.
677
- seq_group = SequenceGroup(request_id=request_id,
678
- seqs=[seq],
679
- arrival_time=arrival_time,
680
- lora_request=lora_request,
681
- pooling_params=pooling_params)
700
+ seq_group = SequenceGroup(
701
+ request_id=request_id,
702
+ seqs=[seq],
703
+ arrival_time=arrival_time,
704
+ lora_request=lora_request,
705
+ pooling_params=pooling_params,
706
+ prompt_adapter_request=prompt_adapter_request)
707
return seq_group
708
709
def abort_request(self, request_id: Union[str, Iterable[str]]) -> None:
@@ -1082,6 +1107,16 @@ class LLMEngine:
1107
def pin_lora(self, lora_id: int) -> bool:
1108
return self.model_executor.pin_lora(lora_id)
1109
1110
+ def add_prompt_adapter(
1111
+ self, prompt_adapter_request: PromptAdapterRequest) -> bool:
1112
+ return self.model_executor.add_prompt_adapter(prompt_adapter_request)
1113
+
1114
+ def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
1115
+ return self.model_executor.remove_prompt_adapter(prompt_adapter_id)
1116
+
1117
+ def list_prompt_adapters(self) -> List[int]:
1118
+ return self.model_executor.list_prompt_adapters()
1119
+
1120
def check_health(self) -> None:
1121
if self.tokenizer:
1122
self.tokenizer.check_health()
vllm/entrypoints/llm.py
+23
-10
@@ -13,6 +13,7 @@ from vllm.logger import init_logger
13
from vllm.lora.request import LoRARequest
14
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
15
from vllm.pooling_params import PoolingParams
16
+from vllm.prompt_adapter.request import PromptAdapterRequest
17
from vllm.sampling_params import SamplingParams
18
from vllm.transformers_utils.tokenizer import get_cached_tokenizer
19
from vllm.usage.usage_lib import UsageContext
@@ -255,6 +256,7 @@ class LLM:
256
prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
257
use_tqdm: bool = True,
258
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
259
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
260
) -> List[RequestOutput]:
261
"""Generates the completions for the input prompts.
262
@@ -271,6 +273,8 @@ class LLM:
273
prompts and it is paired one by one with the prompt.
274
use_tqdm: Whether to use tqdm to display the progress bar.
275
lora_request: LoRA request to use for generation, if any.
276
+ prompt_adapter_request: Prompt Adapter request to use for
277
+ generation, if any.
278
279
Returns:
280
A list of `RequestOutput` objects containing the
@@ -304,7 +308,7 @@ class LLM:
308
inputs=inputs,
309
params=sampling_params,
310
lora_request=lora_request,
307
- )
311
+ prompt_adapter_request=prompt_adapter_request)
312
313
outputs = self._run_engine(use_tqdm=use_tqdm)
314
return LLMEngine.validate_outputs(outputs, RequestOutput)
@@ -397,6 +401,7 @@ class LLM:
401
prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
402
use_tqdm: bool = True,
403
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
404
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
405
) -> List[EmbeddingRequestOutput]:
406
"""Generates the completions for the input prompts.
407
@@ -412,6 +417,8 @@ class LLM:
417
use the default pooling parameters.
418
use_tqdm: Whether to use tqdm to display the progress bar.
419
lora_request: LoRA request to use for generation, if any.
420
+ prompt_adapter_request: Prompt Adapter request to use for
421
+ generation, if any.
422
423
Returns:
424
A list of `EmbeddingRequestOutput` objects containing the
@@ -445,6 +452,7 @@ class LLM:
452
inputs=inputs,
453
params=pooling_params,
454
lora_request=lora_request,
455
+ prompt_adapter_request=prompt_adapter_request,
456
)
457
458
outputs = self._run_engine(use_tqdm=use_tqdm)
@@ -504,6 +512,7 @@ class LLM:
512
params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
513
Sequence[PoolingParams]],
514
lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
515
+ prompt_adapter_request: Optional[PromptAdapterRequest],
516
) -> None:
517
if isinstance(inputs, (str, dict)):
518
# Convert a single prompt to a list.
@@ -526,19 +535,23 @@ class LLM:
535
params[i] if isinstance(params, Sequence) else params,
536
lora_request=lora_request[i] if isinstance(
537
lora_request, Sequence) else lora_request,
529
- )
538
+ prompt_adapter_request=prompt_adapter_request)
539
540
def _add_request(
532
- self,
533
- inputs: PromptInputs,
534
- params: Union[SamplingParams, PoolingParams],
535
- lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
541
+ self,
542
+ inputs: PromptInputs,
543
+ params: Union[SamplingParams, PoolingParams],
544
+ lora_request: Optional[Union[List[LoRARequest],
545
+ LoRARequest]] = None,
546
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None
547
) -> None:
548
request_id = str(next(self.request_counter))
538
- self.llm_engine.add_request(request_id,
539
- inputs,
540
- params,
541
- lora_request=lora_request)
549
+ self.llm_engine.add_request(
550
+ request_id,
551
+ inputs,
552
+ params,
553
+ lora_request=lora_request,
554
+ prompt_adapter_request=prompt_adapter_request)
555
556
def _run_engine(
557
self, *, use_tqdm: bool
vllm/entrypoints/openai/api_server.py
+3
-2
@@ -116,7 +116,7 @@ async def detokenize(request: DetokenizeRequest):
116
117
@app.get("/v1/models")
118
async def show_available_models():
119
- models = await openai_serving_chat.show_available_models()
119
+ models = await openai_serving_completion.show_available_models()
120
return JSONResponse(content=models.model_dump())
121
122
@@ -236,7 +236,8 @@ if __name__ == "__main__":
236
args.lora_modules,
237
args.chat_template)
238
openai_serving_completion = OpenAIServingCompletion(
239
- engine, model_config, served_model_names, args.lora_modules)
239
+ engine, model_config, served_model_names, args.lora_modules,
240
+ args.prompt_adapters)
241
openai_serving_embedding = OpenAIServingEmbedding(engine, model_config,
242
served_model_names)
243
app.root_path = args.root_path
vllm/entrypoints/openai/cli_args.py
+20
-1
@@ -9,7 +9,8 @@ import json
9
import ssl
10
11
from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
12
-from vllm.entrypoints.openai.serving_engine import LoRAModulePath
12
+from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
13
+ PromptAdapterPath)
14
from vllm.utils import FlexibleArgumentParser
15
16
@@ -23,6 +24,16 @@ class LoRAParserAction(argparse.Action):
24
setattr(namespace, self.dest, lora_list)
25
26
27
+class PromptAdapterParserAction(argparse.Action):
28
+
29
+ def __call__(self, parser, namespace, values, option_string=None):
30
+ adapter_list = []
31
+ for item in values:
32
+ name, path = item.split('=')
33
+ adapter_list.append(PromptAdapterPath(name, path))
34
+ setattr(namespace, self.dest, adapter_list)
35
+
36
+
37
def make_arg_parser():
38
parser = FlexibleArgumentParser(
39
description="vLLM OpenAI-Compatible RESTful API server.")
@@ -65,6 +76,14 @@ def make_arg_parser():
76
action=LoRAParserAction,
77
help="LoRA module configurations in the format name=path. "
78
"Multiple modules can be specified.")
79
+ parser.add_argument(
80
+ "--prompt-adapters",
81
+ type=nullable_str,
82
+ default=None,
83
+ nargs='+',
84
+ action=PromptAdapterParserAction,
85
+ help="Prompt adapter configurations in the format name=path. "
86
+ "Multiple adapters can be specified.")
87
parser.add_argument("--chat-template",
88
type=nullable_str,
89
default=None,
vllm/entrypoints/openai/serving_chat.py
+1
-1
@@ -258,7 +258,7 @@ class OpenAIServingChat(OpenAIServing):
258
prompt=prompt,
259
add_special_tokens=request.add_special_tokens)
260
sampling_params = request.to_sampling_params()
261
- lora_request = self._maybe_get_lora(request)
261
+ _, lora_request = self._maybe_get_adapter(request)
262
decoding_config = await self.engine.get_decoding_config()
263
guided_decoding_backend = request.guided_decoding_backend \
264
or decoding_config.guided_decoding_backend
vllm/entrypoints/openai/serving_completion.py
+13
-4
@@ -22,7 +22,8 @@ from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
22
TokenizeResponse, UsageInfo)
23
# yapf: enable
24
from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
25
- OpenAIServing)
25
+ OpenAIServing,
26
+ PromptAdapterPath)
27
from vllm.logger import init_logger
28
from vllm.model_executor.guided_decoding import (
29
get_guided_decoding_logits_processor)
@@ -67,11 +68,13 @@ class OpenAIServingCompletion(OpenAIServing):
68
69
def __init__(self, engine: AsyncLLMEngine, model_config: ModelConfig,
70
served_model_names: List[str],
70
- lora_modules: Optional[List[LoRAModulePath]]):
71
+ lora_modules: Optional[List[LoRAModulePath]],
72
+ prompt_adapters: Optional[List[PromptAdapterPath]]):
73
super().__init__(engine=engine,
74
model_config=model_config,
75
served_model_names=served_model_names,
74
- lora_modules=lora_modules)
76
+ lora_modules=lora_modules,
77
+ prompt_adapters=prompt_adapters)
78
79
async def create_completion(self, request: CompletionRequest,
80
raw_request: Request):
@@ -101,7 +104,12 @@ class OpenAIServingCompletion(OpenAIServing):
104
generators: List[AsyncIterator[RequestOutput]] = []
105
try:
106
sampling_params = request.to_sampling_params()
104
- lora_request = self._maybe_get_lora(request)
107
+ adapter_type, adapter_request = self._maybe_get_adapter(request)
108
+ lora_request, prompt_adapter_request = None, None
109
+ if adapter_type == 'LoRA':
110
+ lora_request, prompt_adapter_request = adapter_request, None
111
+ elif adapter_type == 'PromptAdapter':
112
+ lora_request, prompt_adapter_request = None, adapter_request
113
decoding_config = await self.engine.get_decoding_config()
114
guided_decoding_backend = request.guided_decoding_backend \
115
or decoding_config.guided_decoding_backend
@@ -147,6 +155,7 @@ class OpenAIServingCompletion(OpenAIServing):
155
sampling_params,
156
f"{request_id}-{i}",
157
lora_request=lora_request,
158
+ prompt_adapter_request=prompt_adapter_request,
159
trace_headers=trace_headers,
160
)
161
vllm/entrypoints/openai/serving_engine.py
+51
-10
@@ -16,12 +16,19 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
16
ModelPermission, TokenizeRequest)
17
from vllm.logger import init_logger
18
from vllm.lora.request import LoRARequest
19
+from vllm.prompt_adapter.request import PromptAdapterRequest
20
from vllm.sequence import Logprob
21
from vllm.transformers_utils.tokenizer import get_tokenizer
22
23
logger = init_logger(__name__)
24
25
26
+@dataclass
27
+class PromptAdapterPath:
28
+ name: str
29
+ local_path: str
30
+
31
+
32
@dataclass
33
class LoRAModulePath:
34
name: str
@@ -30,9 +37,14 @@ class LoRAModulePath:
37
38
class OpenAIServing:
39
33
- def __init__(self, engine: AsyncLLMEngine, model_config: ModelConfig,
34
- served_model_names: List[str],
35
- lora_modules: Optional[List[LoRAModulePath]]):
40
+ def __init__(
41
+ self,
42
+ engine: AsyncLLMEngine,
43
+ model_config: ModelConfig,
44
+ served_model_names: List[str],
45
+ lora_modules: Optional[List[LoRAModulePath]],
46
+ prompt_adapters: Optional[List[PromptAdapterPath]] = None,
47
+ ):
48
super().__init__()
49
50
self.engine = engine
@@ -49,9 +61,8 @@ class OpenAIServing:
61
62
self.served_model_names = served_model_names
63
52
- if lora_modules is None:
53
- self.lora_requests = []
54
- else:
64
+ self.lora_requests = []
65
+ if lora_modules is not None:
66
self.lora_requests = [
67
LoRARequest(
68
lora_name=lora.name,
@@ -60,6 +71,20 @@ class OpenAIServing:
71
) for i, lora in enumerate(lora_modules, start=1)
72
]
73
74
+ self.prompt_adapter_requests = []
75
+ if prompt_adapters is not None:
76
+ for i, prompt_adapter in enumerate(prompt_adapters, start=1):
77
+ with open(f"./{prompt_adapter.local_path}"
78
+ f"/adapter_config.json") as f:
79
+ adapter_config = json.load(f)
80
+ num_virtual_tokens = adapter_config["num_virtual_tokens"]
81
+ self.prompt_adapter_requests.append(
82
+ PromptAdapterRequest(
83
+ prompt_adapter_name=prompt_adapter.name,
84
+ prompt_adapter_id=i,
85
+ prompt_adapter_local_path=prompt_adapter.local_path,
86
+ prompt_adapter_num_virtual_tokens=num_virtual_tokens))
87
+
88
async def show_available_models(self) -> ModelList:
89
"""Show available models. Right now we only have one model."""
90
model_cards = [
@@ -75,7 +100,14 @@ class OpenAIServing:
100
permission=[ModelPermission()])
101
for lora in self.lora_requests
102
]
103
+ prompt_adapter_cards = [
104
+ ModelCard(id=prompt_adapter.prompt_adapter_name,
105
+ root=self.served_model_names[0],
106
+ permission=[ModelPermission()])
107
+ for prompt_adapter in self.prompt_adapter_requests
108
+ ]
109
model_cards.extend(lora_cards)
110
+ model_cards.extend(prompt_adapter_cards)
111
return ModelList(data=model_cards)
112
113
def create_error_response(
@@ -109,20 +141,29 @@ class OpenAIServing:
141
return None
142
if request.model in [lora.lora_name for lora in self.lora_requests]:
143
return None
144
+ if request.model in [
145
+ prompt_adapter.prompt_adapter_name
146
+ for prompt_adapter in self.prompt_adapter_requests
147
+ ]:
148
+ return None
149
return self.create_error_response(
150
message=f"The model `{request.model}` does not exist.",
151
err_type="NotFoundError",
152
status_code=HTTPStatus.NOT_FOUND)
153
117
- def _maybe_get_lora(
154
+ def _maybe_get_adapter(
155
self, request: Union[CompletionRequest, ChatCompletionRequest,
156
EmbeddingRequest]
120
- ) -> Optional[LoRARequest]:
157
+ ) -> Tuple[Optional[str], Optional[Union[LoRARequest,
158
+ PromptAdapterRequest]]]:
159
if request.model in self.served_model_names:
122
- return None
160
+ return None, None
161
for lora in self.lora_requests:
162
if request.model == lora.lora_name:
125
- return lora
163
+ return 'LoRA', lora
164
+ for prompt_adapter in self.prompt_adapter_requests:
165
+ if request.model == prompt_adapter.prompt_adapter_name:
166
+ return 'PromptAdapter', prompt_adapter
167
# if _check_model has been called earlier, this will be unreachable
168
raise ValueError(f"The model `{request.model}` does not exist.")
169
vllm/executor/cpu_executor.py
+15
@@ -7,6 +7,7 @@ from vllm.config import CacheConfig, ModelConfig, SchedulerConfig
7
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
8
from vllm.logger import init_logger
9
from vllm.lora.request import LoRARequest
10
+from vllm.prompt_adapter.request import PromptAdapterRequest
11
from vllm.sequence import ExecuteModelRequest, SamplerOutput
12
from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
13
make_async)
@@ -48,6 +49,7 @@ class CPUExecutor(ExecutorBase):
49
lora_config=self.lora_config,
50
multimodal_config=self.multimodal_config,
51
kv_cache_dtype=self.cache_config.cache_dtype,
52
+ prompt_adapter_config=self.prompt_adapter_config,
53
is_driver_worker=True,
54
)
55
self.driver_worker.init_device()
@@ -90,6 +92,19 @@ class CPUExecutor(ExecutorBase):
92
def list_loras(self) -> Set[int]:
93
return self.driver_worker.list_loras()
94
95
+ def add_prompt_adapter(
96
+ self, prompt_adapter_request: PromptAdapterRequest) -> bool:
97
+ return self.driver_worker.add_prompt_adapter(prompt_adapter_request)
98
+
99
+ def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
100
+ return self.driver_worker.remove_prompt_adapter(prompt_adapter_id)
101
+
102
+ def list_prompt_adapters(self) -> Set[int]:
103
+ return self.driver_worker.list_prompt_adapters()
104
+
105
+ def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
106
+ return self.driver_worker.pin_prompt_adapter(prompt_adapter_id)
107
+
108
def check_health(self) -> None:
109
# CPUExecutor will always be healthy as long as
110
# it's running.
vllm/executor/executor_base.py
+25
-2
@@ -4,8 +4,10 @@ from typing import List, Optional, Set, Tuple
4
5
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6
ModelConfig, MultiModalConfig, ParallelConfig,
7
- SchedulerConfig, SpeculativeConfig)
7
+ PromptAdapterConfig, SchedulerConfig,
8
+ SpeculativeConfig)
9
from vllm.lora.request import LoRARequest
10
+from vllm.prompt_adapter.request import PromptAdapterRequest
11
from vllm.sequence import ExecuteModelRequest, SamplerOutput
12
13
@@ -28,6 +30,7 @@ class ExecutorBase(ABC):
30
lora_config: Optional[LoRAConfig],
31
multimodal_config: Optional[MultiModalConfig],
32
speculative_config: Optional[SpeculativeConfig],
33
+ prompt_adapter_config: Optional[PromptAdapterConfig],
34
) -> None:
35
self.model_config = model_config
36
self.cache_config = cache_config
@@ -38,6 +41,7 @@ class ExecutorBase(ABC):
41
self.device_config = device_config
42
self.multimodal_config = multimodal_config
43
self.speculative_config = speculative_config
44
+ self.prompt_adapter_config = prompt_adapter_config
45
46
self._init_executor()
47
@@ -95,6 +99,23 @@ class ExecutorBase(ABC):
99
def list_loras(self) -> Set[int]:
100
raise NotImplementedError
101
102
+ @abstractmethod
103
+ def add_prompt_adapter(
104
+ self, prompt_adapter_request: PromptAdapterRequest) -> bool:
105
+ raise NotImplementedError
106
+
107
+ @abstractmethod
108
+ def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
109
+ raise NotImplementedError
110
+
111
+ @abstractmethod
112
+ def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
113
+ raise NotImplementedError # type: ignore
114
+
115
+ @abstractmethod
116
+ def list_prompt_adapters(self) -> Set[int]:
117
+ raise NotImplementedError
118
+
119
@abstractmethod
120
def check_health(self) -> None:
121
"""Checks if the executor is healthy. If not, it should raise an
@@ -122,12 +143,14 @@ class ExecutorAsyncBase(ExecutorBase):
143
lora_config: Optional[LoRAConfig],
144
multimodal_config: Optional[MultiModalConfig],
145
speculative_config: Optional[SpeculativeConfig],
146
+ prompt_adapter_config: Optional[PromptAdapterConfig],
147
) -> None:
148
self.pp_locks: Optional[List[asyncio.Lock]] = None
149
150
super().__init__(model_config, cache_config, parallel_config,
151
scheduler_config, device_config, load_config,
130
- lora_config, multimodal_config, speculative_config)
152
+ lora_config, multimodal_config, speculative_config,
153
+ prompt_adapter_config)
154
155
@abstractmethod
156
async def execute_model_async(
vllm/executor/gpu_executor.py
+21
@@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional, Set, Tuple, Union
3
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4
from vllm.logger import init_logger
5
from vllm.lora.request import LoRARequest
6
+from vllm.prompt_adapter.request import PromptAdapterRequest
7
from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
8
from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
9
make_async)
@@ -45,6 +46,7 @@ class GPUExecutor(ExecutorBase):
46
lora_config=self.lora_config,
47
multimodal_config=self.multimodal_config,
48
speculative_config=self.speculative_config,
49
+ prompt_adapter_config=self.prompt_adapter_config,
50
is_driver_worker=(not self.parallel_config)
51
or (rank % self.parallel_config.tensor_parallel_size == 0),
52
)
@@ -107,6 +109,25 @@ class GPUExecutor(ExecutorBase):
109
def list_loras(self) -> Set[int]:
110
return self.driver_worker.list_loras()
111
112
+ def add_prompt_adapter(
113
+ self, prompt_adapter_request: PromptAdapterRequest) -> bool:
114
+ assert prompt_adapter_request.prompt_adapter_id > 0, \
115
+ "prompt_adapter_id must be greater than 0."
116
+ return self.driver_worker.add_prompt_adapter(prompt_adapter_request)
117
+
118
+ def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
119
+ assert prompt_adapter_id > 0, \
120
+ "prompt_adapter_id must be greater than 0."
121
+ return self.driver_worker.remove_prompt_adapter(prompt_adapter_id)
122
+
123
+ def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
124
+ assert prompt_adapter_id > 0, \
125
+ "prompt_adapter_id must be greater than 0."
126
+ return self.driver_worker.pin_prompt_adapter(prompt_adapter_id)
127
+
128
+ def list_prompt_adapters(self) -> Set[int]:
129
+ return self.driver_worker.list_prompt_adapters()
130
+
131
def check_health(self) -> None:
132
# GPUExecutor will always be healthy as long as
133
# it's running.
vllm/executor/ray_xpu_executor.py
+4
-1
@@ -8,7 +8,8 @@ from typing import (TYPE_CHECKING, Any, Awaitable, Dict, List, Optional, Set,
8
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
ModelConfig, MultiModalConfig, ParallelConfig,
11
- SchedulerConfig, SpeculativeConfig)
11
+ PromptAdapterConfig, SchedulerConfig,
12
+ SpeculativeConfig)
13
from vllm.executor.distributed_gpu_executor import ( # yapf: disable
14
DistributedGPUExecutor, DistributedGPUExecutorAsync)
15
from vllm.executor.ray_utils import RayWorkerWrapper, ray
@@ -44,6 +45,7 @@ class RayXPUExecutor(DistributedGPUExecutor):
45
load_config: LoadConfig,
46
lora_config: Optional[LoRAConfig],
47
multimodal_config: Optional[MultiModalConfig],
48
+ prompt_adapter_config: Optional[PromptAdapterConfig],
49
speculative_config: Optional[SpeculativeConfig],
50
) -> None:
51
assert device_config.device_type == "xpu"
@@ -58,6 +60,7 @@ class RayXPUExecutor(DistributedGPUExecutor):
60
self.scheduler_config = scheduler_config
61
self.device_config = device_config
62
self.multimodal_config = multimodal_config
63
+ self.prompt_adapter_config = prompt_adapter_config
64
65
placement_group = self.parallel_config.placement_group
66
vllm/executor/xpu_executor.py
+4
-1
@@ -4,7 +4,8 @@ import torch
4
5
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6
ModelConfig, MultiModalConfig, ParallelConfig,
7
- SchedulerConfig, SpeculativeConfig)
7
+ PromptAdapterConfig, SchedulerConfig,
8
+ SpeculativeConfig)
9
from vllm.executor.executor_base import ExecutorAsyncBase
10
from vllm.executor.gpu_executor import GPUExecutor
11
from vllm.logger import init_logger
@@ -27,6 +28,7 @@ class XPUExecutor(GPUExecutor):
28
load_config: LoadConfig,
29
lora_config: Optional[LoRAConfig],
30
multimodal_config: Optional[MultiModalConfig],
31
+ prompt_adapter_config: Optional[PromptAdapterConfig],
32
speculative_config: Optional[SpeculativeConfig],
33
) -> None:
34
assert device_config.device_type == "xpu"
@@ -43,6 +45,7 @@ class XPUExecutor(GPUExecutor):
45
self.scheduler_config = scheduler_config
46
self.device_config = device_config
47
self.multimodal_config = multimodal_config
48
+ self.prompt_adapter_config = prompt_adapter_config
49
self.speculative_config = None
50
51
# Instantiate the worker and load the model to GPU.
vllm/lora/layers.py
+3
-9
@@ -8,6 +8,7 @@ import torch.nn as nn
8
import torch.nn.functional as F
9
from transformers import PretrainedConfig
10
11
+from vllm.adapter_commons.layers import AdapterMapping
12
from vllm.config import LoRAConfig
13
from vllm.distributed import (get_tensor_model_parallel_rank,
14
get_tensor_model_parallel_world_size,
@@ -134,15 +135,8 @@ def _apply_lora_packed_nslice(
135
136
137
@dataclass
137
-class LoRAMapping:
138
- # Per every token in input_ids:
139
- index_mapping: Tuple[int, ...]
140
- # Per sampled token:
141
- prompt_mapping: Tuple[int, ...]
142
-
143
- def __post_init__(self):
144
- self.index_mapping = tuple(self.index_mapping)
145
- self.prompt_mapping = tuple(self.prompt_mapping)
138
+class LoRAMapping(AdapterMapping):
139
+ pass
140
141
142
class BaseLayerWithLoRA(nn.Module):
vllm/lora/models.py
+80
-95
@@ -4,12 +4,17 @@ import math
4
import os
5
import re
6
from dataclasses import dataclass, field
7
-from typing import Callable, Dict, List, Optional, Tuple, Type, Union
7
+from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
8
9
import safetensors.torch
10
import torch
11
from torch import nn
12
13
+from vllm.adapter_commons.models import (AdapterLRUCache, AdapterModel,
14
+ AdapterModelManager)
15
+from vllm.adapter_commons.utils import (add_adapter, deactivate_adapter,
16
+ get_adapter, list_adapters,
17
+ remove_adapter, set_adapter_mapping)
18
from vllm.config import LoRAConfig
19
from vllm.logger import init_logger
20
from vllm.lora.layers import (BaseLayerWithLoRA,
@@ -19,7 +24,7 @@ from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
24
from vllm.lora.utils import (from_layer, from_layer_logits_processor,
25
parse_fine_tuned_lora_name, replace_submodule)
26
from vllm.model_executor.models.interfaces import SupportsLoRA
22
-from vllm.utils import LRUCache, is_pin_memory_available
27
+from vllm.utils import is_pin_memory_available
28
29
logger = init_logger(__name__)
30
@@ -153,7 +158,7 @@ def get_lora_id():
158
return _GLOBAL_LORA_ID
159
160
156
-class LoRAModel:
161
+class LoRAModel(AdapterModel):
162
"""A LoRA fine-tuned model."""
163
164
def __init__(
@@ -388,7 +393,7 @@ class LoRAModel:
393
)
394
395
391
-class LoRAModelManager:
396
+class LoRAModelManager(AdapterModelManager):
397
"""A manager that manages multiple LoRA-fine-tuned models."""
398
399
def __init__(
@@ -440,8 +445,7 @@ class LoRAModelManager:
445
# base_indices, sampler_indices, sampler_indices_padded,
446
# embeddings_indices
447
self.indices_len: List[Optional[int]] = [None] * 4
443
-
444
- self.model = model
448
+ super().__init__(model)
449
if hasattr(self.model, "supported_lora_modules"):
450
self.supported_lora_modules = copy.deepcopy(
451
self.model.supported_lora_modules)
@@ -453,11 +457,11 @@ class LoRAModelManager:
457
self.model.packed_modules_mapping)
458
self.packed_modules: Dict[str, List[str]] = {}
459
self.modules: Dict[str, "BaseLayerWithLoRA"] = {}
456
- self._registered_loras: Dict[int, LoRAModel] = {}
460
# Dict instead of a Set for compatibility with LRUCache.
458
- self._active_loras: Dict[int, None] = {}
461
self._last_mapping: Optional[LoRAMapping] = None
462
self._create_lora_modules()
463
+ self.model.lora_manager = self
464
+ self.adapter_type = 'LoRa'
465
466
@property
467
def capacity(self) -> int:
@@ -467,15 +471,16 @@ class LoRAModelManager:
471
def lora_slots(self) -> int:
472
return self.lora_config.max_loras
473
470
- def __len__(self) -> int:
471
- return len(self._registered_loras)
474
+ @property
475
+ def adapter_slots(self) -> int:
476
+ return self.lora_slots
477
473
- def activate_lora(
478
+ def activate_adapter(
479
self,
480
lora_id: int,
481
) -> bool:
482
"""Move LoRA into a GPU buffer to be used in the forward pass."""
478
- if lora_id in self._active_loras:
483
+ if lora_id in self._active_adapters:
484
return False
485
first_free_slot = next(
486
((i, lora_id) for i, lora_id in enumerate(self.lora_index_to_id)
@@ -483,8 +488,8 @@ class LoRAModelManager:
488
if first_free_slot is None:
489
raise ValueError("No free lora slots")
490
index, _ = first_free_slot
486
- self._active_loras[lora_id] = None
487
- lora_model = self._registered_loras[lora_id]
491
+ self._active_adapters[lora_id] = None
492
+ lora_model = self._registered_adapters[lora_id]
493
logger.debug("Activating LoRA. int id: %d, slot index: %d",
494
lora_model.id, index)
495
self.lora_index_to_id[index] = lora_model.id
@@ -498,21 +503,13 @@ class LoRAModelManager:
503
module.reset_lora(index)
504
return True
505
501
- def _deactivate_lora(self, lora_id: int):
506
+ def _deactivate_adapter(self, lora_id: int):
507
try:
508
index = self.lora_index_to_id.index(lora_id)
509
self.lora_index_to_id[index] = None
510
except ValueError:
511
pass
512
508
- def deactivate_lora(self, lora_id: int) -> bool:
509
- """Remove a LoRA from a GPU buffer."""
510
- if lora_id in self._active_loras:
511
- self._deactivate_lora(lora_id)
512
- self._active_loras.pop(lora_id)
513
- return True
514
- return False
515
-
513
def _set_long_lora_context(self, lora: LoRAModel):
514
if self.long_lora_context is None:
515
return
@@ -528,40 +525,19 @@ class LoRAModelManager:
525
if offsets:
526
self.long_lora_context.offsets_by_lora_id[lora.id] = offsets
527
531
- def _add_lora(self, lora: LoRAModel):
528
+ def _add_adapter(self, lora: LoRAModel):
529
self._create_merged_loras_inplace(lora)
533
- self._registered_loras[lora.id] = lora
530
+ self._registered_adapters[lora.id] = lora
531
self._set_long_lora_context(lora)
532
536
- def add_lora(self, lora: LoRAModel) -> bool:
537
- """Add a LoRAModel to the manager CPU cache."""
538
- logger.debug(
539
- "Adding lora. Model id: %d, "
540
- "int id: %d, "
541
- "scaling factor: %s", lora.id, lora.id, lora.scaling_factor)
542
- if lora.id not in self._registered_loras:
543
- if len(self._registered_loras) >= self.capacity:
544
- raise RuntimeError("No free LoRA slots.")
545
- self._add_lora(lora)
546
- return True
547
- return False
548
-
549
- def remove_lora(self, lora_id: int) -> bool:
550
- """Remove a LoRAModel from the manager CPU cache."""
551
- # TODO: should we check active lora?
552
- self.deactivate_lora(lora_id)
553
- if self.long_lora_context:
554
- self.long_lora_context.offsets_by_lora_id.pop(lora_id, None)
555
- return bool(self._registered_loras.pop(lora_id, None))
556
-
557
- def pin_lora(self, lora_id: int) -> bool:
533
+ def pin_adapter(self, lora_id: int) -> bool:
534
"""Pin a LoRAModel in the manager cache."""
535
raise NotImplementedError(
536
"Pinning is not supported in LoRAModelManager."
537
"Use LRUCacheLoRAModelManager for pinning") # type: ignore
538
539
# TODO see if this can be vectorized
564
- def _set_lora_mapping(self, mapping: LoRAMapping) -> None:
540
+ def _set_adapter_mapping(self, mapping: LoRAMapping) -> None:
541
(base_indices, sampler_indices, sampler_indices_padded,
542
embeddings_indices, long_lora_offsets_tensor,
543
indices_len) = convert_mapping(mapping, self.lora_index_to_id,
@@ -583,23 +559,11 @@ class LoRAModelManager:
559
# Maintain the reference
560
self.indices_len[:] = indices_len
561
586
- def set_lora_mapping(self, lora_mapping: LoRAMapping) -> None:
587
- if self._last_mapping != lora_mapping:
588
- self._set_lora_mapping(lora_mapping)
589
- self._last_mapping = lora_mapping
590
-
591
- def list_loras(self) -> Dict[int, LoRAModel]:
592
- """List all registered LoRAModels."""
593
- return dict(self._registered_loras)
594
-
595
- def get_lora(self, lora_id: int) -> Optional[LoRAModel]:
596
- return self._registered_loras.get(lora_id, None)
597
-
598
- def remove_all_loras(self):
562
+ def remove_all_adapters(self):
563
"""Remove all LoRAModels from the manager."""
600
- self._registered_loras.clear()
564
+ self._registered_adapters.clear()
565
self.lora_index_to_id = [None] * self.lora_slots
602
- self._active_loras.clear()
566
+ self._active_adapters.clear()
567
568
def _create_lora_modules(self):
569
for module_name, module in self.model.named_modules(
@@ -743,18 +707,39 @@ class LoRAModelManager:
707
lora_model.loras[module_name] = PackedLoRALayerWeights.pack(
708
replacement_loras)
709
710
+ def deactivate_adapter(self, adapter_id: int) -> bool:
711
+ return deactivate_adapter(adapter_id, self._active_adapters,
712
+ self._deactivate_adapter)
713
+
714
+ def add_adapter(self, adapter: LoRAModel) -> bool:
715
+ logger.debug(
716
+ "Adding lora. Model id: %d, "
717
+ "int id: %d, "
718
+ "scaling factor: %s", adapter.id, adapter.id,
719
+ adapter.scaling_factor)
720
+ return add_adapter(adapter, self._registered_adapters, self.capacity,
721
+ self._add_adapter)
722
747
-class LoRALRUCache(LRUCache[LoRAModel]):
723
+ def set_adapter_mapping(self, mapping: LoRAMapping) -> None:
724
+ self._last_mapping = set_adapter_mapping(mapping, self._last_mapping,
725
+ self._set_adapter_mapping)
726
+
727
+ def remove_adapter(self, adapter_id: int) -> bool:
728
+ return remove_adapter(adapter_id, self._registered_adapters,
729
+ self.deactivate_adapter)
730
+
731
+ def list_adapters(self) -> Dict[int, Any]:
732
+ return list_adapters(self._registered_adapters)
733
+
734
+ def get_adapter(self, adapter_id: int) -> Optional[Any]:
735
+ return get_adapter(adapter_id, self._registered_adapters)
736
+
737
+
738
+class LoRALRUCache(AdapterLRUCache[LoRAModel]):
739
740
def __init__(self, capacity: int, deactivate_lora_fn: Callable[[int],
741
bool]):
751
- super().__init__(capacity)
752
- self.deactivate_lora_fn = deactivate_lora_fn
753
-
754
- def _on_remove(self, key: int, value: LoRAModel):
755
- logger.debug("Removing LoRA. int id: %d", key)
756
- self.deactivate_lora_fn(key)
757
- return super()._on_remove(key, value)
742
+ super().__init__(capacity, deactivate_lora_fn)
743
744
745
class LRUCacheLoRAModelManager(LoRAModelManager):
@@ -770,49 +755,49 @@ class LRUCacheLoRAModelManager(LoRAModelManager):
755
):
756
super().__init__(model, max_num_seqs, max_num_batched_tokens,
757
vocab_size, lora_config)
773
- self._registered_loras: LoRALRUCache = LoRALRUCache(
774
- self.capacity, self.deactivate_lora)
775
- self._active_loras: LoRALRUCache = LoRALRUCache(
776
- self.lora_slots, self._deactivate_lora)
758
+ self._registered_adapters: LoRALRUCache = LoRALRUCache(
759
+ self.capacity, self.deactivate_adapter)
760
+ self._active_adapters: LoRALRUCache = LoRALRUCache(
761
+ self.lora_slots, self._deactivate_adapter)
762
778
- def list_loras(self) -> Dict[int, LoRAModel]:
763
+ def list_adapters(self) -> Dict[int, LoRAModel]:
764
"""List all registered LoRAModels."""
780
- return dict(self._registered_loras.cache)
765
+ return dict(self._registered_adapters.cache)
766
782
- def add_lora(self, lora: LoRAModel) -> bool:
767
+ def add_adapter(self, lora: LoRAModel) -> bool:
768
"""Add a LoRAModel to the manager."""
769
logger.debug(
770
"Adding lora. Model id: %d, "
771
"int id: %d, "
772
"scaling factor: %s", lora.id, lora.id, lora.scaling_factor)
788
- if lora.id not in self._registered_loras:
789
- self._add_lora(lora)
773
+ if lora.id not in self._registered_adapters:
774
+ self._add_adapter(lora)
775
was_added = True
776
else:
777
# We always touch to update the LRU cache order
793
- self._registered_loras.touch(lora.id)
778
+ self._registered_adapters.touch(lora.id)
779
was_added = False
780
return was_added
781
797
- def activate_lora(
782
+ def activate_adapter(
783
self,
784
lora_id: int,
785
) -> bool:
801
- if lora_id not in self._active_loras and len(
802
- self._active_loras) >= self.lora_slots:
803
- self._active_loras.remove_oldest()
804
- result = super().activate_lora(lora_id)
786
+ if lora_id not in self._active_adapters and len(
787
+ self._active_adapters) >= self.lora_slots:
788
+ self._active_adapters.remove_oldest()
789
+ result = super().activate_adapter(lora_id)
790
# We always touch to update the LRU cache order
806
- self._active_loras.touch(lora_id)
791
+ self._active_adapters.touch(lora_id)
792
return result
793
809
- def remove_oldest_lora(self) -> bool:
810
- if len(self._registered_loras) > 0:
811
- self._registered_loras.remove_oldest()
794
+ def remove_oldest_adapter(self) -> bool:
795
+ if len(self._registered_adapters) > 0:
796
+ self._registered_adapters.remove_oldest()
797
return True
798
return False
799
815
- def pin_lora(self, lora_id: int) -> bool:
800
+ def pin_adapter(self, lora_id: int) -> bool:
801
"""Pin a LoRAModel in the manager cache."""
802
self._pin_lora_in_cpu_cache(lora_id)
803
self._pin_lora_in_gpu_cache(lora_id)
@@ -820,17 +805,17 @@ class LRUCacheLoRAModelManager(LoRAModelManager):
805
806
def _pin_lora_in_cpu_cache(self, lora_id: int):
807
try:
823
- self._registered_loras.pin(lora_id)
808
+ self._registered_adapters.pin(lora_id)
809
except ValueError as err:
810
raise ValueError("Pinning failed. "
811
f"LoRA {lora_id} is not registered.") from err
812
813
def _pin_lora_in_gpu_cache(self, lora_id: int):
829
- if lora_id not in self._active_loras:
814
+ if lora_id not in self._active_adapters:
815
# move lora to gpu if not already active
831
- self.activate_lora(lora_id)
816
+ self.activate_adapter(lora_id)
817
833
- self._active_loras.pin(lora_id)
818
+ self._active_adapters.pin(lora_id)
819
820
821
def create_lora_manager(
vllm/lora/request.py
+14
-11
@@ -1,13 +1,15 @@
1
from dataclasses import dataclass
2
from typing import Optional
3
4
+from vllm.adapter_commons.request import AdapterRequest
5
+
6
7
@dataclass
6
-class LoRARequest:
8
+class LoRARequest(AdapterRequest):
9
"""
10
Request for a LoRA adapter.
11
10
- Note that this class should be be used internally. For online
12
+ Note that this class should be used internally. For online
13
serving, it is recommended to not allow users to use this class but
14
instead provide another layer of abstraction to prevent users from
15
accessing unauthorized LoRA adapters.
@@ -20,15 +22,16 @@ class LoRARequest:
22
lora_int_id: int
23
lora_local_path: str
24
long_lora_max_len: Optional[int] = None
25
+ __hash__ = AdapterRequest.__hash__
26
24
- def __post_init__(self):
25
- if self.lora_int_id < 1:
26
- raise ValueError(
27
- f"lora_int_id must be > 0, got {self.lora_int_id}")
27
+ @property
28
+ def adapter_id(self):
29
+ return self.lora_int_id
30
29
- def __eq__(self, value: object) -> bool:
30
- return isinstance(
31
- value, LoRARequest) and self.lora_int_id == value.lora_int_id
31
+ @property
32
+ def name(self):
33
+ return self.lora_name
34
33
- def __hash__(self) -> int:
34
- return self.lora_int_id
35
+ @property
36
+ def local_path(self):
37
+ return self.lora_local_path
vllm/lora/worker_manager.py
+70
-145
@@ -1,12 +1,15 @@
1
-from abc import ABC, abstractmethod
1
from contextlib import contextmanager
2
from typing import Any, Dict, List, Literal, Optional, Set, Type, Union
3
4
import torch
5
6
+from vllm.adapter_commons.utils import (add_adapter_worker,
7
+ apply_adapters_worker,
8
+ list_adapters_worker,
9
+ set_active_adapters_worker)
10
+from vllm.adapter_commons.worker_manager import AbstractWorkerManager
11
from vllm.config import LoRAConfig
12
from vllm.logger import init_logger
9
-from vllm.lora.layers import LoRAMapping
13
from vllm.lora.models import (LoRAModel, LoRAModelManager,
14
LRUCacheLoRAModelManager, create_lora_manager)
15
from vllm.lora.request import LoRARequest
@@ -14,79 +17,13 @@ from vllm.lora.request import LoRARequest
17
logger = init_logger(__name__)
18
19
17
-class AbstractWorkerLoRAManager(ABC):
18
- """Abstract class for managing LoRA models on the worker side."""
19
-
20
- def __init__(self,
21
- max_num_seqs: int,
22
- max_num_batched_tokens: int,
23
- vocab_size: int,
24
- lora_config: LoRAConfig,
25
- device: torch.device,
26
- max_position_embeddings: Optional[int] = None):
27
- self.max_num_seqs = max_num_seqs
28
- self.max_num_batched_tokens = max_num_batched_tokens
29
- self.max_position_embeddings = max_position_embeddings
30
- self.vocab_size = vocab_size
31
- self.device = device
32
- self.lora_config = lora_config
33
-
34
- # If False, do not cache. If None, cache is empty.
35
- self._cached_dummy_lora: Union[None, Literal[False], LoRAModel] = False
36
-
37
- @contextmanager
38
- def dummy_lora_cache(self):
39
- """Use this context manager to reuse the dummy lora model
40
- to avoid creating it repeatedly."""
41
- self._cached_dummy_lora = None
42
- yield
43
- self._cached_dummy_lora = False
44
-
45
- @property
46
- @abstractmethod
47
- def is_enabled(self) -> bool:
48
- ...
49
-
50
- @abstractmethod
51
- def create_lora_manager(
52
- self,
53
- model: torch.nn.Module,
54
- ) -> Any:
55
- ...
56
-
57
- @abstractmethod
58
- def set_active_loras(self, lora_requests: Set[LoRARequest],
59
- lora_mapping: LoRAMapping) -> None:
60
- ...
61
-
62
- @abstractmethod
63
- def add_lora(self, lora_request: LoRARequest) -> bool:
64
- ...
65
-
66
- @abstractmethod
67
- def add_dummy_lora(self, lora_request: LoRARequest, rank: int) -> bool:
68
- ...
69
-
70
- @abstractmethod
71
- def remove_lora(self, lora_id: int) -> bool:
72
- ...
73
-
74
- @abstractmethod
75
- def remove_all_loras(self):
76
- ...
77
-
78
- @abstractmethod
79
- def list_loras(self) -> Set[int]:
80
- ...
81
-
82
-
83
-class WorkerLoRAManager(AbstractWorkerLoRAManager):
20
+class WorkerLoRAManager(AbstractWorkerManager):
21
"""WorkerLoRAManager that manages LoRA models on the worker side.
22
23
Every request, the requested LoRAs will be loaded (unless they are already
24
loaded), and every other LoRA will be unloaded."""
25
89
- _lora_manager_cls: Type[LoRAModelManager] = LoRAModelManager
26
+ _manager_cls: Type[LoRAModelManager] = LoRAModelManager
27
28
def __init__(
29
self,
@@ -103,16 +40,23 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
40
self._lora_model_cls = lora_model_cls
41
self.embedding_modules = embedding_modules
42
self.embedding_padding_modules = embedding_padding_modules
43
+ self._cached_dummy_lora: Union[None, Literal[False], LoRAModel] = False
44
+ self.max_num_seqs = max_num_seqs
45
+ self.max_num_batched_tokens = max_num_batched_tokens
46
+ self.vocab_size = vocab_size
47
+ self.lora_config = lora_config
48
+ self.max_position_embeddings = max_position_embeddings
49
+ super().__init__(device)
50
# Lazily initialized by create_lora_manager.
107
- self._lora_manager: LoRAModelManager
108
- super().__init__(
109
- max_num_seqs,
110
- max_num_batched_tokens,
111
- vocab_size,
112
- lora_config,
113
- device,
114
- max_position_embeddings=max_position_embeddings,
115
- )
51
+ self._adapter_manager: LoRAModelManager
52
+
53
+ @contextmanager
54
+ def dummy_lora_cache(self):
55
+ """Use this context manager to reuse the dummy lora model
56
+ to avoid creating it repeatedly."""
57
+ self._cached_dummy_lora = None
58
+ yield
59
+ self._cached_dummy_lora = False
60
61
@property
62
def is_enabled(self) -> bool:
@@ -128,41 +72,14 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
72
max_num_batched_tokens=self.max_num_batched_tokens,
73
vocab_size=self.vocab_size,
74
lora_config=self.lora_config,
131
- lora_manager_cls=self._lora_manager_cls,
75
+ lora_manager_cls=self._manager_cls,
76
)
133
- self._lora_manager = lora_manager
77
+ self._adapter_manager = lora_manager
78
return lora_manager.model
79
136
- def set_active_loras(self, lora_requests: Set[LoRARequest],
137
- lora_mapping: LoRAMapping) -> None:
138
- self._apply_loras(lora_requests)
139
- self._lora_manager.set_lora_mapping(lora_mapping)
140
-
141
- def _apply_loras(self, lora_requests: Set[LoRARequest]) -> None:
142
- loras_that_exist = self.list_loras()
143
- loras_map = {
144
- lora_request.lora_int_id: lora_request
145
- for lora_request in lora_requests if lora_request
146
- }
147
- if len(loras_map) > self._lora_manager.lora_slots:
148
- raise RuntimeError(
149
- f"Number of requested LoRAs ({len(loras_map)}) is greater "
150
- "than the number of GPU LoRA slots "
151
- f"({self._lora_manager.lora_slots}).")
152
-
153
- new_loras = set(loras_map)
154
- loras_to_add = new_loras - loras_that_exist
155
- loras_to_remove = loras_that_exist - new_loras
156
-
157
- for lora_id in loras_to_remove:
158
- self.remove_lora(lora_id)
159
-
160
- for lora_id in loras_to_add:
161
- self.add_lora(loras_map[lora_id])
162
-
163
- def _load_lora(self, lora_request: LoRARequest) -> LoRAModel:
80
+ def _load_adapter(self, lora_request: LoRARequest) -> LoRAModel:
81
try:
165
- model = self._lora_manager.model
82
+ model = self._adapter_manager.model
83
supported_lora_modules = model.supported_lora_modules
84
packed_modules_mapping = model.packed_modules_mapping
85
expected_lora_modules: List[str] = []
@@ -198,37 +115,45 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
115
return lora
116
117
def add_dummy_lora(self, lora_request: LoRARequest, rank: int) -> bool:
201
- if lora_request.lora_int_id in self.list_loras():
118
+ if lora_request.lora_int_id in self.list_adapters():
119
return False
120
if isinstance(self._cached_dummy_lora, LoRAModel):
121
dummy_lora = self._cached_dummy_lora.clone(
122
lora_request.lora_int_id)
123
else:
207
- dummy_lora = self._lora_manager.create_dummy_lora(
124
+ dummy_lora = self._adapter_manager.create_dummy_lora(
125
lora_request.lora_int_id, rank, 1, self.embedding_modules)
126
if self._cached_dummy_lora is None:
127
self._cached_dummy_lora = dummy_lora
211
- return self._lora_manager.add_lora(dummy_lora)
128
+ return self._adapter_manager.add_adapter(dummy_lora)
129
213
- def add_lora(self, lora_request: LoRARequest) -> bool:
214
- if lora_request.lora_int_id in self.list_loras():
215
- return False
216
- lora = self._load_lora(lora_request)
217
- loaded = self._lora_manager.add_lora(lora)
218
- self._lora_manager.activate_lora(lora.id)
219
- return loaded
130
+ def pin_adapter(self, adapter_id: int) -> bool:
131
+ return self._adapter_manager.pin_adapter(adapter_id)
132
+
133
+ def set_active_adapters(self, requests: Set[Any],
134
+ mapping: Optional[Any]) -> None:
135
+ set_active_adapters_worker(requests, mapping, self._apply_adapters,
136
+ self._adapter_manager.set_adapter_mapping)
137
+
138
+ def _apply_adapters(self, adapter_requests: Set[Any]) -> None:
139
+ apply_adapters_worker(adapter_requests, self.list_adapters,
140
+ self._adapter_manager.adapter_slots,
141
+ self.remove_adapter, self.add_adapter)
142
221
- def remove_lora(self, lora_id: int) -> bool:
222
- return self._lora_manager.remove_lora(lora_id)
143
+ def add_adapter(self, adapter_request: Any) -> bool:
144
+ return add_adapter_worker(adapter_request, self.list_adapters,
145
+ self._load_adapter,
146
+ self._adapter_manager.add_adapter,
147
+ self._adapter_manager.activate_adapter)
148
224
- def pin_lora(self, lora_id: int) -> bool:
225
- return self._lora_manager.pin_lora(lora_id)
149
+ def remove_adapter(self, adapter_id: int) -> bool:
150
+ return self._adapter_manager.remove_adapter(adapter_id)
151
227
- def remove_all_loras(self):
228
- self._lora_manager.remove_all_loras()
152
+ def remove_all_adapters(self):
153
+ self._adapter_manager.remove_all_adapters()
154
230
- def list_loras(self) -> Set[int]:
231
- return set(self._lora_manager.list_loras())
155
+ def list_adapters(self) -> Set[int]:
156
+ return list_adapters_worker(self._adapter_manager.list_adapters)
157
158
159
class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
@@ -238,8 +163,7 @@ class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
163
(unless they are already loaded) and least recently used LoRAs will
164
be unloaded if the cache is above capacity."""
165
241
- _lora_manager_cls: Type[
242
- LRUCacheLoRAModelManager] = LRUCacheLoRAModelManager
166
+ _manager_cls: Type[LRUCacheLoRAModelManager] = LRUCacheLoRAModelManager
167
168
def create_lora_manager(
169
self,
@@ -247,40 +171,41 @@ class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
171
) -> Any:
172
lora_manager = create_lora_manager(
173
model,
250
- lora_manager_cls=self._lora_manager_cls,
174
+ lora_manager_cls=self._manager_cls,
175
max_num_seqs=self.max_num_seqs,
176
vocab_size=self.vocab_size,
177
lora_config=self.lora_config,
178
max_num_batched_tokens=self.max_num_batched_tokens,
179
)
256
- self._lora_manager = lora_manager
180
+ self._adapter_manager = lora_manager
181
return lora_manager.model
182
259
- def _apply_loras(self, lora_requests: Set[LoRARequest]) -> None:
183
+ def _apply_adapters(self, lora_requests: Set[LoRARequest]) -> None:
184
loras_map = {
185
lora_request.lora_int_id: lora_request
186
for lora_request in lora_requests if lora_request
187
}
264
- if len(loras_map) > self._lora_manager.lora_slots:
188
+ if len(loras_map) > self._adapter_manager.lora_slots:
189
raise RuntimeError(
190
f"Number of requested LoRAs ({len(loras_map)}) is greater "
191
"than the number of GPU LoRA slots "
268
- f"({self._lora_manager.lora_slots}).")
192
+ f"({self._adapter_manager.lora_slots}).")
193
for lora in loras_map.values():
270
- self.add_lora(lora)
194
+ self.add_adapter(lora)
195
272
- def add_lora(self, lora_request: LoRARequest) -> bool:
273
- if lora_request.lora_int_id not in self.list_loras():
196
+ def add_adapter(self, lora_request: LoRARequest) -> bool:
197
+ if lora_request.lora_int_id not in self.list_adapters():
198
# Remove before we load the new lora to save memory
275
- if len(self._lora_manager) + 1 > self._lora_manager.capacity:
276
- assert isinstance(self._lora_manager, LRUCacheLoRAModelManager)
277
- self._lora_manager.remove_oldest_lora()
278
- lora = self._load_lora(lora_request)
279
- loaded = self._lora_manager.add_lora(lora)
199
+ if len(self._adapter_manager) + 1 > self._adapter_manager.capacity:
200
+ assert isinstance(self._adapter_manager,
201
+ LRUCacheLoRAModelManager)
202
+ self._adapter_manager.remove_oldest_adapter()
203
+ lora = self._load_adapter(lora_request)
204
+ loaded = self._adapter_manager.add_adapter(lora)
205
else:
206
# If the lora is already loaded, just touch it to
207
# update its position in the caches
283
- loaded = self._lora_manager.get_lora(
208
+ loaded = self._adapter_manager.get_adapter(
209
lora_request.lora_int_id) is not None
285
- self._lora_manager.activate_lora(lora_request.lora_int_id)
210
+ self._adapter_manager.activate_adapter(lora_request.lora_int_id)
211
return loaded
vllm/prompt_adapter/__init__.py
vllm/prompt_adapter/layers.py
new
+80
@@ -0,0 +1,80 @@
1
+from dataclasses import dataclass
2
+from typing import Optional
3
+
4
+import torch
5
+from torch import nn
6
+
7
+from vllm.adapter_commons.layers import AdapterMapping
8
+from vllm.config import PromptAdapterConfig
9
+from vllm.model_executor.layers.vocab_parallel_embedding import (
10
+ VocabParallelEmbedding)
11
+
12
+
13
+@dataclass
14
+class PromptAdapterMapping(AdapterMapping):
15
+ pass
16
+
17
+
18
+class VocabParallelEmbeddingWithPromptAdapter(nn.Module):
19
+
20
+ def __init__(self, base_layer: VocabParallelEmbedding) -> None:
21
+ super().__init__()
22
+ self.base_layer = base_layer
23
+ self.emb_layer = self.base_layer
24
+ if 'LoRA' in base_layer.__class__.__name__:
25
+ self.emb_layer = self.base_layer.base_layer
26
+
27
+ def create_prompt_adapter_weights(
28
+ self, prompt_adapter_config: PromptAdapterConfig):
29
+ self.embeddings_tensors = torch.zeros(
30
+ (
31
+ prompt_adapter_config.max_prompt_adapters,
32
+ prompt_adapter_config.max_prompt_adapter_token,
33
+ self.emb_layer.embedding_dim,
34
+ ),
35
+ dtype=self.emb_layer.weight.dtype,
36
+ device=self.emb_layer.weight.device,
37
+ )
38
+ self.adapter_lengths = torch.zeros(
39
+ prompt_adapter_config.max_prompt_adapters,
40
+ dtype=torch.long,
41
+ device=self.emb_layer.weight.device)
42
+
43
+ self.indices_gpu: torch.Tensor
44
+ self.embedding_indices_gpu: torch.Tensor
45
+
46
+ def reset_prompt_adapter(self, index: int):
47
+ self.embeddings_tensors[index] = 0
48
+
49
+ def set_prompt_adapter(
50
+ self,
51
+ index: int,
52
+ adapter_model: Optional[torch.Tensor],
53
+ ):
54
+ self.reset_prompt_adapter(index)
55
+ if adapter_model is not None:
56
+ length = adapter_model.shape[0]
57
+ self.embeddings_tensors[index, :length] = adapter_model
58
+ self.adapter_lengths[index] = length
59
+
60
+ def set_mapping(
61
+ self,
62
+ prompt_indices: torch.Tensor,
63
+ prompt_embedding_indices: torch.Tensor,
64
+ ):
65
+ self.indices_gpu = prompt_indices.to(
66
+ device=self.emb_layer.weight.device)
67
+ self.embedding_indices_gpu = prompt_embedding_indices.to(
68
+ device=self.emb_layer.weight.device)
69
+
70
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
71
+ hidden_states = self.base_layer(x)
72
+ if self.embedding_indices_gpu.ndim > 1:
73
+ valid_mask = self.indices_gpu != -1
74
+ gathered_embeddings = self.embeddings_tensors[
75
+ self.embedding_indices_gpu[:, 0],
76
+ self.embedding_indices_gpu[:, 1]]
77
+
78
+ # Update hidden states
79
+ hidden_states[valid_mask] = gathered_embeddings
80
+ return hidden_states
\ No newline at end of file
vllm/prompt_adapter/models.py
new
+355
@@ -0,0 +1,355 @@
1
+import logging
2
+import math
3
+from typing import Any, Callable, Dict, List, Optional, Type
4
+
5
+import torch
6
+from torch import nn
7
+
8
+from vllm.adapter_commons.models import (AdapterLRUCache, AdapterModel,
9
+ AdapterModelManager)
10
+from vllm.adapter_commons.utils import (add_adapter, deactivate_adapter,
11
+ get_adapter, list_adapters,
12
+ remove_adapter, set_adapter_mapping)
13
+from vllm.config import PromptAdapterConfig
14
+from vllm.prompt_adapter.layers import (
15
+ VocabParallelEmbeddingWithPromptAdapter) # yapf: disable
16
+from vllm.prompt_adapter.layers import PromptAdapterMapping
17
+
18
+logger = logging.getLogger(__name__)
19
+
20
+_GLOBAL_PROMPT_ADAPTER_ID = 0
21
+
22
+
23
+def get_prompt_adapter_id():
24
+ global _GLOBAL_PROMPT_ADAPTER_ID
25
+ _GLOBAL_PROMPT_ADAPTER_ID += 1
26
+ return _GLOBAL_PROMPT_ADAPTER_ID
27
+
28
+
29
+def convert_to_embedding_indices(indices):
30
+ embedding_indices = []
31
+ count = 0
32
+
33
+ for value in indices:
34
+ if value == -1:
35
+ count = 0
36
+ else:
37
+ embedding_indices.append([value, count])
38
+ count += 1
39
+
40
+ return torch.tensor(embedding_indices)
41
+
42
+
43
+def convert_mapping(
44
+ mapping: PromptAdapterMapping,
45
+ prompt_adapter_index_to_id: List[Optional[int]],
46
+) -> torch.Tensor:
47
+ """Converts PromptAdapterMapping to index tensors.
48
+
49
+ Args:
50
+ mapping: PromptAdapterMapping mapping rows in a
51
+ batch to PromptAdapter ids.
52
+ prompt_adapter_index_to_id: List mapping PromptAdapter
53
+ ids to PromptAdapter indices.
54
+
55
+ Returns:
56
+ pa_indices: Tensor of shape [batch_size] mapping batch rows to
57
+ PromptAdapter indices.
58
+ """
59
+ id_to_index = {
60
+ id_: idx
61
+ for idx, id_ in enumerate(prompt_adapter_index_to_id)
62
+ if id_ is not None
63
+ }
64
+ pa_indices = ([
65
+ id_to_index.get(id_, -1) if id_ > 0 else -1
66
+ for id_ in mapping.index_mapping
67
+ ])
68
+
69
+ pa_embedding_mapping = convert_to_embedding_indices(pa_indices)
70
+ pa_indices = torch.tensor(pa_indices)
71
+ return pa_indices, pa_embedding_mapping
72
+
73
+
74
+class PromptAdapterModel(AdapterModel):
75
+
76
+ def __init__(self,
77
+ prompt_adapter_id=None,
78
+ num_virtual_tokens=None,
79
+ prompt_embedding=None) -> None:
80
+ self.id = prompt_adapter_id
81
+ self.prompt_embedding = prompt_embedding
82
+ self.num_virtual_tokens = num_virtual_tokens
83
+
84
+ @classmethod
85
+ def from_local_checkpoint(
86
+ cls,
87
+ adapter_model_path: str,
88
+ prompt_adapter_id: int,
89
+ num_virtual_tokens: int,
90
+ config: PromptAdapterConfig,
91
+ device: str = "cuda",
92
+ ) -> "PromptAdapterModel":
93
+ from peft.utils import load_peft_weights
94
+
95
+ if num_virtual_tokens > config.max_prompt_adapter_token:
96
+ raise ValueError(
97
+ f'num_virtual_tokens ({num_virtual_tokens}) should be <= '
98
+ f'max_prompt_adapter_token({config.max_prompt_adapter_token})')
99
+
100
+ adapters_weights = load_peft_weights(adapter_model_path, device)
101
+ prompt_embedding = adapters_weights["prompt_embeddings"].to(
102
+ config.prompt_adapter_dtype)
103
+
104
+ return cls(prompt_adapter_id, num_virtual_tokens, prompt_embedding)
105
+
106
+
107
+class PromptAdapterModelManager(AdapterModelManager):
108
+ """A manager that manages multiple Prompt Adapter models."""
109
+
110
+ def __init__(
111
+ self,
112
+ model: nn.Module,
113
+ max_num_seqs: int,
114
+ max_num_batched_tokens: int,
115
+ prompt_adapter_config: PromptAdapterConfig,
116
+ ):
117
+ """Create a PromptAdapterModel and adapter for a given model.
118
+
119
+ Args:
120
+ model: the model to be adapted.
121
+ max_num_seqs: the maximum number of sequences model can run in a
122
+ single batch.
123
+ max_num_batched_tokens: the maximum number of tokens model can run
124
+ in a single batch.
125
+ prompt_adapter_config: the PromptAdapter config,
126
+ """
127
+ self.model: nn.Module = model
128
+ # Dict instead of a Set for compatibility with LRUCache.
129
+ self.prompt_adapter_index_to_id: List[
130
+ Optional[int]] = [None] * self.prompt_adapter_slots
131
+ self.max_num_seqs = max_num_seqs
132
+ self.max_num_batched_tokens = math.ceil(max_num_batched_tokens / 8) * 8
133
+ self.prompt_adapter_config = prompt_adapter_config
134
+ self.model.prompt_adapter_manager = self
135
+ self.adapter_type = 'PromptAdapter'
136
+
137
+ self.base_indices = torch.tensor([-1])
138
+ self.base_embedding_indices = torch.tensor([])
139
+
140
+ self.modules: Dict[str, nn.Module] = {}
141
+ self._create_prompt_adapter_modules()
142
+ self._last_mapping: Optional[PromptAdapterMapping] = None
143
+
144
+ @property
145
+ def prompt_adapter_slots(self) -> int:
146
+ return self.prompt_adapter_config.max_prompt_adapters
147
+
148
+ @property
149
+ def adapter_slots(self) -> int:
150
+ return self.prompt_adapter_slots
151
+
152
+ @property
153
+ def capacity(self) -> int:
154
+ return self.prompt_adapter_config.max_cpu_prompt_adapters
155
+
156
+ def activate_adapter(
157
+ self,
158
+ prompt_adapter_id: int,
159
+ ) -> bool:
160
+ """Move PromptAdapter into a GPU buffer
161
+ to be used in the forward pass."""
162
+ if prompt_adapter_id in self._active_adapters:
163
+ return False
164
+ first_free_slot = next(
165
+ ((i, prompt_adapter_id) for i, prompt_adapter_id in enumerate(
166
+ self.prompt_adapter_index_to_id) if prompt_adapter_id is None),
167
+ None)
168
+ if first_free_slot is None:
169
+ raise ValueError("No free prompt_adapter slots")
170
+ index, _ = first_free_slot
171
+ self._active_adapters[prompt_adapter_id] = None
172
+ prompt_adapter_model = (self._registered_adapters[prompt_adapter_id])
173
+ logger.debug("Activating prompt_adapter. int id: %d, slot index: %d",
174
+ prompt_adapter_model.id, index)
175
+ self.prompt_adapter_index_to_id[index] = prompt_adapter_model.id
176
+ for _, v in self.modules.items():
177
+ v.set_prompt_adapter(index, prompt_adapter_model.prompt_embedding)
178
+ return True
179
+
180
+ def _deactivate_adapter(self, prompt_adapter_id: int):
181
+ try:
182
+ index = self.prompt_adapter_index_to_id.index(prompt_adapter_id)
183
+ self.prompt_adapter_index_to_id[index] = None
184
+ for _, v in self.modules.items():
185
+ v.reset_prompt_adapter(index)
186
+ except ValueError:
187
+ pass
188
+
189
+ def _add_adapter(self, prompt_adapter: PromptAdapterModel):
190
+ self._registered_adapters[prompt_adapter.id] = prompt_adapter
191
+
192
+ def _set_adapter_mapping(self, mapping: PromptAdapterMapping) -> None:
193
+ base_indices, base_embedding_indices = convert_mapping(
194
+ mapping, self.prompt_adapter_index_to_id)
195
+ for k, v in self.modules.items():
196
+ v.set_mapping(base_indices, base_embedding_indices)
197
+
198
+ def _create_prompt_adapter_modules(self):
199
+ for module_name, module in self.model.named_modules(
200
+ remove_duplicate=False):
201
+ if "VocabParallel" in module.__class__.__name__:
202
+ new_module = VocabParallelEmbeddingWithPromptAdapter(module)
203
+ new_module.create_prompt_adapter_weights(
204
+ self.prompt_adapter_config)
205
+ replaced_module = self.replace_submodule(
206
+ self.model, module_name, new_module)
207
+ self.register_module(module.__class__.__name__,
208
+ replaced_module)
209
+ replaced_module.set_mapping(self.base_indices,
210
+ self.base_embedding_indices)
211
+ break
212
+
213
+ def replace_submodule(self, model: nn.Module, module_name: str,
214
+ new_module: nn.Module) -> nn.Module:
215
+ """Replace a submodule in a model with a new module."""
216
+ parent = model.get_submodule(".".join(module_name.split(".")[:-1]))
217
+ target_name = module_name.split(".")[-1]
218
+ setattr(parent, target_name, new_module)
219
+ return new_module
220
+
221
+ def register_module(self, module_name: str, module: nn.Module):
222
+ self.modules[module_name] = module
223
+
224
+ def pin_adapter(self, prompt_adapter_id: int) -> bool:
225
+ """Pin a PromptAdapterModel in the manager cache."""
226
+ raise NotImplementedError(
227
+ "Pinning is not supported in PromptAdapterModelManager."
228
+ "Use LRUCachePromptAdapterModelManager for pinning"
229
+ ) # type: ignore
230
+
231
+ def remove_all_adapters(self):
232
+ """Remove all PromptAdapterModel from the manager."""
233
+ self._registered_adapters.clear()
234
+ self.prompt_adapter_index_to_id = [None] * self.prompt_adapter_slots
235
+ self._active_adapters.clear()
236
+
237
+ def deactivate_adapter(self, adapter_id: int) -> bool:
238
+ return deactivate_adapter(adapter_id, self._active_adapters,
239
+ self._deactivate_adapter)
240
+
241
+ def add_adapter(self, adapter: PromptAdapterModel) -> bool:
242
+ return add_adapter(adapter, self._registered_adapters, self.capacity,
243
+ self._add_adapter)
244
+
245
+ def set_adapter_mapping(self, mapping: PromptAdapterMapping) -> None:
246
+ self._last_mapping = set_adapter_mapping(mapping, self._last_mapping,
247
+ self._set_adapter_mapping)
248
+
249
+ def remove_adapter(self, adapter_id: int) -> bool:
250
+ return remove_adapter(adapter_id, self._registered_adapters,
251
+ self.deactivate_adapter)
252
+
253
+ def list_adapters(self) -> Dict[int, Any]:
254
+ return list_adapters(self._registered_adapters)
255
+
256
+ def get_adapter(self, adapter_id: int) -> Optional[Any]:
257
+ return get_adapter(adapter_id, self._registered_adapters)
258
+
259
+
260
+class PromptAdapterLRUCache(AdapterLRUCache[PromptAdapterModel]):
261
+
262
+ def __init__(self, capacity: int,
263
+ deactivate_prompt_adapter_fn: Callable[[int], bool]):
264
+ super().__init__(capacity, deactivate_prompt_adapter_fn)
265
+
266
+
267
+class LRUCachePromptAdapterModelManager(PromptAdapterModelManager):
268
+ """A model manager that manages multiple prompt_adapters with LRU cache."""
269
+
270
+ def __init__(
271
+ self,
272
+ model: nn.Module,
273
+ max_num_seqs: int,
274
+ max_num_batched_tokens: int,
275
+ prompt_adapter_config: PromptAdapterConfig,
276
+ ):
277
+ self.prompt_adapter_config = prompt_adapter_config
278
+ super().__init__(model, max_num_seqs, max_num_batched_tokens,
279
+ prompt_adapter_config)
280
+ self._registered_adapters = PromptAdapterLRUCache(
281
+ self.capacity, self.deactivate_adapter)
282
+ self._active_adapters = PromptAdapterLRUCache(
283
+ self.prompt_adapter_slots, self._deactivate_adapter)
284
+
285
+ def list_adapters(self) -> Dict[int, PromptAdapterModel]:
286
+ """List all registered PromptAdapterModel."""
287
+ return dict(self._registered_adapters.cache)
288
+
289
+ def add_adapter(self, prompt_adapter: PromptAdapterModel) -> bool:
290
+ """Add a PromptAdapterModel to the manager."""
291
+ if prompt_adapter.id not in self._registered_adapters:
292
+ self._add_adapter(prompt_adapter)
293
+ was_added = True
294
+ else:
295
+ # We always touch to update the LRU cache order
296
+ self._registered_adapters.touch(prompt_adapter.id)
297
+ was_added = False
298
+ return was_added
299
+
300
+ def activate_adapter(
301
+ self,
302
+ prompt_adapter_id: int,
303
+ ) -> bool:
304
+ if prompt_adapter_id not in self._active_adapters and len(
305
+ self._active_adapters) >= self.prompt_adapter_slots:
306
+ self._active_adapters.remove_oldest()
307
+ result = super().activate_adapter(prompt_adapter_id)
308
+ # We always touch to update the LRU cache order
309
+ self._active_adapters.touch(prompt_adapter_id)
310
+ return result
311
+
312
+ def remove_oldest_adapter(self) -> bool:
313
+ if len(self._registered_adapters) > 0:
314
+ self._registered_adapters.remove_oldest()
315
+ return True
316
+ return False
317
+
318
+ def pin_adapter(self, prompt_adapter_id: int) -> bool:
319
+ """Pin a PromptAdapterModel in the manager cache."""
320
+ self._pin_prompt_adapter_in_cpu_cache(prompt_adapter_id)
321
+ self._pin_prompt_adapter_in_gpu_cache(prompt_adapter_id)
322
+ return True
323
+
324
+ def _pin_prompt_adapter_in_cpu_cache(self, prompt_adapter_id: int):
325
+ try:
326
+ self._registered_adapters.pin(prompt_adapter_id)
327
+ except ValueError as err:
328
+ raise ValueError(
329
+ "Pinning failed. "
330
+ f"Prompt Adapter {prompt_adapter_id} is not registered."
331
+ ) from err
332
+
333
+ def _pin_prompt_adapter_in_gpu_cache(self, prompt_adapter_id: int):
334
+ if prompt_adapter_id not in self._active_adapters:
335
+ # move adapter to gpu if not already active
336
+ self.activate_adapter(prompt_adapter_id)
337
+ self._active_adapters.pin(prompt_adapter_id)
338
+
339
+
340
+def create_prompt_adapter_manager(
341
+ model: nn.Module,
342
+ max_num_seqs: int,
343
+ max_num_batched_tokens: int,
344
+ prompt_adapter_config: PromptAdapterConfig,
345
+ prompt_adapter_manager_cls: Type[
346
+ PromptAdapterModelManager] = PromptAdapterModelManager,
347
+ **kwargs) -> PromptAdapterModelManager:
348
+ """Create a PromptAdapterModel for a given model."""
349
+ prompt_adapter_manager = prompt_adapter_manager_cls(
350
+ model=model,
351
+ max_num_seqs=max_num_seqs,
352
+ max_num_batched_tokens=max_num_batched_tokens,
353
+ prompt_adapter_config=prompt_adapter_config,
354
+ **kwargs)
355
+ return prompt_adapter_manager
vllm/prompt_adapter/request.py
new
+30
@@ -0,0 +1,30 @@
1
+from dataclasses import dataclass
2
+
3
+from vllm.adapter_commons.request import AdapterRequest
4
+
5
+
6
+@dataclass
7
+class PromptAdapterRequest(AdapterRequest):
8
+ """
9
+ Request for a Prompt adapter.
10
+ """
11
+
12
+ prompt_adapter_name: str
13
+ prompt_adapter_id: int
14
+ prompt_adapter_local_path: str
15
+ prompt_adapter_num_virtual_tokens: int
16
+
17
+ def __hash__(self):
18
+ return super().__hash__()
19
+
20
+ @property
21
+ def adapter_id(self):
22
+ return self.prompt_adapter_id
23
+
24
+ @property
25
+ def name(self):
26
+ return self.prompt_adapter_name
27
+
28
+ @property
29
+ def local_path(self):
30
+ return self.prompt_adapter_local_path
vllm/prompt_adapter/worker_manager.py
new
+176
@@ -0,0 +1,176 @@
1
+import logging
2
+from typing import Any, Optional, Set, Type
3
+
4
+import torch
5
+
6
+from vllm.adapter_commons.utils import (add_adapter_worker,
7
+ apply_adapters_worker,
8
+ list_adapters_worker,
9
+ set_active_adapters_worker)
10
+from vllm.adapter_commons.worker_manager import AbstractWorkerManager
11
+from vllm.config import PromptAdapterConfig
12
+from vllm.prompt_adapter.models import (LRUCachePromptAdapterModelManager,
13
+ PromptAdapterModel,
14
+ PromptAdapterModelManager,
15
+ create_prompt_adapter_manager)
16
+from vllm.prompt_adapter.request import PromptAdapterRequest
17
+
18
+logger = logging.getLogger(__name__)
19
+
20
+
21
+class WorkerPromptAdapterManager(AbstractWorkerManager):
22
+ """WorkerPromptAdapterManager that manages
23
+ prompt_adapter models on the worker side.
24
+
25
+ Every request, the requested prompt_adapters will be
26
+ loaded (unless they are already loaded),
27
+ and every other prompt_adapter will be unloaded."""
28
+
29
+ _manager_cls: Type[PromptAdapterModelManager] = PromptAdapterModelManager
30
+
31
+ def __init__(
32
+ self,
33
+ max_num_seqs: int,
34
+ max_num_batched_tokens: int,
35
+ device: torch.device,
36
+ prompt_adapter_config: PromptAdapterConfig,
37
+ prompt_adapter_model_cls: Type[PromptAdapterModel] = PromptAdapterModel
38
+ ):
39
+ self._adapter_manager: PromptAdapterModelManager
40
+ self.max_num_seqs = max_num_seqs
41
+ self.max_num_batched_tokens = max_num_batched_tokens
42
+ self._prompt_adapter_model_cls = prompt_adapter_model_cls
43
+ self.prompt_adapter_config = prompt_adapter_config
44
+ super().__init__(device)
45
+
46
+ @property
47
+ def is_enabled(self) -> bool:
48
+ return True
49
+
50
+ def create_prompt_adapter_manager(
51
+ self,
52
+ model: torch.nn.Module,
53
+ ) -> Any:
54
+ prompt_adapter_manager = create_prompt_adapter_manager(
55
+ model,
56
+ max_num_seqs=self.max_num_seqs,
57
+ max_num_batched_tokens=self.max_num_batched_tokens,
58
+ prompt_adapter_config=self.prompt_adapter_config,
59
+ prompt_adapter_manager_cls=self._manager_cls,
60
+ )
61
+ self._adapter_manager = prompt_adapter_manager
62
+ return prompt_adapter_manager.model
63
+
64
+ def _load_adapter(
65
+ self, prompt_adapter_request: PromptAdapterRequest
66
+ ) -> PromptAdapterModel:
67
+ try:
68
+ prompt_adapter = (
69
+ self._prompt_adapter_model_cls.from_local_checkpoint(
70
+ prompt_adapter_request.prompt_adapter_local_path,
71
+ prompt_adapter_id=prompt_adapter_request.prompt_adapter_id,
72
+ num_virtual_tokens=prompt_adapter_request.
73
+ prompt_adapter_num_virtual_tokens,
74
+ config=self.prompt_adapter_config,
75
+ device=str(self.device),
76
+ ))
77
+ except Exception as e:
78
+ raise RuntimeError(
79
+ f"Loading prompt_adapter "
80
+ f"{prompt_adapter_request.prompt_adapter_local_path}"
81
+ f" failed") from e
82
+ return prompt_adapter
83
+
84
+ def add_dummy_prompt_adapter(
85
+ self, prompt_adapter_request: PromptAdapterRequest) -> bool:
86
+ return True
87
+
88
+ def pin_adapter(self, adapter_id: int) -> bool:
89
+ return self._adapter_manager.pin_adapter(adapter_id)
90
+
91
+ def set_active_adapters(self, requests: Set[Any],
92
+ mapping: Optional[Any]) -> None:
93
+ set_active_adapters_worker(requests, mapping, self._apply_adapters,
94
+ self._adapter_manager.set_adapter_mapping)
95
+
96
+ def add_adapter(self, adapter_request: Any) -> bool:
97
+ return add_adapter_worker(adapter_request, self.list_adapters,
98
+ self._load_adapter,
99
+ self._adapter_manager.add_adapter,
100
+ self._adapter_manager.activate_adapter)
101
+
102
+ def _apply_adapters(self, adapter_requests: Set[Any]) -> None:
103
+ apply_adapters_worker(adapter_requests, self.list_adapters,
104
+ self._adapter_manager.adapter_slots,
105
+ self.remove_adapter, self.add_adapter)
106
+
107
+ def remove_adapter(self, adapter_id: int) -> bool:
108
+ return self._adapter_manager.remove_adapter(adapter_id)
109
+
110
+ def remove_all_adapters(self):
111
+ self._adapter_manager.remove_all_adapters()
112
+
113
+ def list_adapters(self) -> Set[int]:
114
+ return list_adapters_worker(self._adapter_manager.list_adapters)
115
+
116
+
117
+class LRUCacheWorkerPromptAdapterManager(WorkerPromptAdapterManager):
118
+ """WorkerPromptAdapterManager that manages
119
+ prompt_adapter models on the worker side.
120
+
121
+ Uses an LRU Cache. Every request, the requested
122
+ prompt_adapters will be loaded (unless they are already loaded)
123
+ and least recently used prompt_adapters will
124
+ be unloaded if the cache is above capacity."""
125
+
126
+ _prompt_adapter_manager_cls: Type[
127
+ LRUCachePromptAdapterModelManager] = LRUCachePromptAdapterModelManager
128
+
129
+ def create_prompt_adapter_manager(
130
+ self,
131
+ model: torch.nn.Module,
132
+ ) -> Any:
133
+ prompt_adapter_manager = create_prompt_adapter_manager(
134
+ model,
135
+ max_num_seqs=self.max_num_seqs,
136
+ max_num_batched_tokens=self.max_num_batched_tokens,
137
+ prompt_adapter_config=self.prompt_adapter_config,
138
+ prompt_adapter_manager_cls=self._prompt_adapter_manager_cls)
139
+ self._adapter_manager: LRUCachePromptAdapterModelManager = (
140
+ prompt_adapter_manager)
141
+ return prompt_adapter_manager.model
142
+
143
+ def _apply_adapters(
144
+ self, prompt_adapter_requests: Set[PromptAdapterRequest]) -> None:
145
+ prompt_adapters_map = {
146
+ prompt_adapter_request.prompt_adapter_id: prompt_adapter_request
147
+ for prompt_adapter_request in prompt_adapter_requests
148
+ if prompt_adapter_request
149
+ }
150
+ if len(prompt_adapters_map
151
+ ) > self._adapter_manager.prompt_adapter_slots:
152
+ raise RuntimeError(
153
+ f"Number of requested prompt_adapters "
154
+ f"({len(prompt_adapters_map)}) is greater "
155
+ "than the number of GPU prompt_adapter slots "
156
+ f"({self._adapter_manager.prompt_adapter_slots}).")
157
+ for prompt_adapter in prompt_adapters_map.values():
158
+ self.add_adapter(prompt_adapter)
159
+
160
+ def add_adapter(self,
161
+ prompt_adapter_request: PromptAdapterRequest) -> bool:
162
+ if prompt_adapter_request.prompt_adapter_id not in self.list_adapters(
163
+ ):
164
+ # Remove before we load the new prompt_adapter to save memory
165
+ if len(self._adapter_manager) + 1 > self._adapter_manager.capacity:
166
+ self._adapter_manager.remove_oldest_adapter()
167
+ prompt_adapter = self._load_adapter(prompt_adapter_request)
168
+ loaded = self._adapter_manager.add_adapter(prompt_adapter)
169
+ else:
170
+ # If the prompt_adapter is already loaded, just touch it to
171
+ # update its position in the caches
172
+ loaded = self._adapter_manager.get_adapter(
173
+ prompt_adapter_request.prompt_adapter_id) is not None
174
+ self._adapter_manager.activate_adapter(
175
+ prompt_adapter_request.prompt_adapter_id)
176
+ return loaded
vllm/sequence.py
+42
-6
@@ -10,6 +10,7 @@ import torch
10
11
from vllm.lora.request import LoRARequest
12
from vllm.pooling_params import PoolingParams
13
+from vllm.prompt_adapter.request import PromptAdapterRequest
14
from vllm.sampling_params import SamplingParams
15
16
if TYPE_CHECKING:
@@ -238,21 +239,25 @@ class Sequence:
239
block_size: The block size of the sequence. Should be the same as the
240
block size used by the block manager and cache engine.
241
lora_request: LoRA request.
242
+ prompt_adapter_request: Prompt Adapter request.
243
+
244
"""
245
246
def __init__(
244
- self,
245
- seq_id: int,
246
- inputs: "LLMInputs",
247
- block_size: int,
248
- eos_token_id: Optional[int] = None,
249
- lora_request: Optional[LoRARequest] = None,
247
+ self,
248
+ seq_id: int,
249
+ inputs: "LLMInputs",
250
+ block_size: int,
251
+ eos_token_id: Optional[int] = None,
252
+ lora_request: Optional[LoRARequest] = None,
253
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None
254
) -> None:
255
self.seq_id = seq_id
256
self.inputs = inputs
257
self.block_size = block_size
258
self.eos_token_id = eos_token_id
259
self.lora_request = lora_request
260
+ self.prompt_adapter_request = prompt_adapter_request
261
262
self.data = SequenceData(self.prompt_token_ids)
263
self.output_logprobs: SampleLogprobs = []
@@ -287,6 +292,11 @@ class Sequence:
292
def lora_int_id(self) -> int:
293
return self.lora_request.lora_int_id if self.lora_request else 0
294
295
+ @property
296
+ def prompt_adapter_id(self) -> int:
297
+ return self.prompt_adapter_request.prompt_adapter_id \
298
+ if self.prompt_adapter_request else 0
299
+
300
def get_output_text_to_return(self, buffer_length: int):
301
# We return the full output text if the sequence is finished.
302
truncate = buffer_length and not self.is_finished()
@@ -414,6 +424,7 @@ class SequenceGroup:
424
encoder_seq: Optional, the single encoder sequence. Should be None
425
unless you are working with an encoder/decoder model.
426
trace_headers: OpenTelemetry trace headers.
427
+ prompt_adapter_request: Prompt Adapter request.
428
"""
429
430
def __init__(
@@ -427,6 +438,7 @@ class SequenceGroup:
438
pooling_params: Optional[PoolingParams] = None,
439
encoder_seq: Optional[Sequence] = None,
440
trace_headers: Optional[Dict[str, str]] = None,
441
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
442
) -> None:
443
self.request_id = request_id
444
self.seqs_dict = {seq.seq_id: seq for seq in seqs}
@@ -441,6 +453,7 @@ class SequenceGroup:
453
self.state = SequenceGroupState()
454
self.embeddings = embeddings
455
self.pooling_params = pooling_params
456
+ self.prompt_adapter_request = prompt_adapter_request
457
self.encoder_seq = encoder_seq
458
self.trace_headers = trace_headers
459
@@ -466,6 +479,16 @@ class SequenceGroup:
479
def lora_int_id(self) -> int:
480
return self.lora_request.lora_int_id if self.lora_request else 0
481
482
+ @property
483
+ def prompt_adapter_id(self) -> int:
484
+ return self.prompt_adapter_request.prompt_adapter_id \
485
+ if self.prompt_adapter_request else 0
486
+
487
+ @property
488
+ def prompt_adapter_num_virtual_tokens(self) -> int:
489
+ return self.prompt_adapter_request.prompt_adapter_num_virtual_tokens\
490
+ if self.prompt_adapter_request else 0
491
+
492
def get_last_latency(self, now: float) -> Optional[float]:
493
"""Sets the last token time for Request level timings."""
494
# If still in prefill phase, raise Error.
@@ -624,6 +647,7 @@ class SequenceGroupMetadata:
647
(SequenceGroup.encoder_seq). Should be None
648
unless you are working with an encoder/decoder
649
model.
650
+ prompt_adapter_request: Prompt Adapter request.
651
"""
652
653
def __init__(
@@ -642,6 +666,7 @@ class SequenceGroupMetadata:
666
multi_modal_data: Optional["MultiModalDataDict"] = None,
667
encoder_seq_data: Optional[SequenceData] = None,
668
cross_block_table: Optional[List[int]] = None,
669
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
670
) -> None:
671
self.request_id = request_id
672
self.is_prompt = is_prompt
@@ -650,6 +675,7 @@ class SequenceGroupMetadata:
675
self.block_tables = block_tables
676
self.pooling_params = pooling_params
677
self.lora_request = lora_request
678
+ self.prompt_adapter_request = prompt_adapter_request
679
self.computed_block_nums = computed_block_nums
680
self.multi_modal_data = multi_modal_data
681
self.state = SequenceGroupState() if state is None else state
@@ -674,6 +700,16 @@ class SequenceGroupMetadata:
700
def lora_int_id(self) -> int:
701
return self.lora_request.lora_int_id if self.lora_request else 0
702
703
+ @property
704
+ def prompt_adapter_id(self) -> int:
705
+ return self.prompt_adapter_request.prompt_adapter_id \
706
+ if self.prompt_adapter_request else 0
707
+
708
+ @property
709
+ def prompt_adapter_num_virtual_tokens(self) -> int:
710
+ return self.prompt_adapter_request.prompt_adapter_num_virtual_tokens \
711
+ if self.prompt_adapter_request else 0
712
+
713
@property
714
def token_chunk_size(self) -> int:
715
"""Return the number of tokens to be processed (chunk size)."""
vllm/spec_decode/draft_model_runner.py
+10
-1
@@ -4,7 +4,7 @@ import torch
4
5
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6
ModelConfig, MultiModalConfig, ParallelConfig,
7
- SchedulerConfig)
7
+ PromptAdapterConfig, SchedulerConfig)
8
from vllm.logger import init_logger
9
from vllm.sequence import (IntermediateTensors, SamplerOutput,
10
SequenceGroupMetadata)
@@ -48,6 +48,7 @@ class TP1DraftModelRunner(ModelRunner):
48
kv_cache_dtype: Optional[str] = "auto",
49
is_driver_worker: bool = False,
50
multimodal_config: Optional[MultiModalConfig] = None,
51
+ prompt_adapter_config: Optional[PromptAdapterConfig] = None,
52
return_hidden_states: bool = False,
53
):
54
if return_hidden_states:
@@ -66,6 +67,7 @@ class TP1DraftModelRunner(ModelRunner):
67
kv_cache_dtype=kv_cache_dtype,
68
is_driver_worker=is_driver_worker,
69
multimodal_config=multimodal_config,
70
+ prompt_adapter_config=prompt_adapter_config,
71
return_hidden_states=return_hidden_states,
72
)
73
@@ -136,6 +138,13 @@ class TP1DraftModelRunner(ModelRunner):
138
self.set_active_loras(model_input.lora_requests,
139
model_input.lora_mapping)
140
141
+ if self.prompt_adapter_config:
142
+ assert model_input.prompt_adapter_requests is not None
143
+ assert model_input.prompt_adapter_mapping is not None
144
+ self.set_active_prompt_adapters(
145
+ model_input.prompt_adapter_requests,
146
+ model_input.prompt_adapter_mapping)
147
+
148
virtual_engine = model_input.virtual_engine
149
outputs: List[SamplerOutput] = []
150
for step in range(num_steps):
vllm/worker/cpu_model_runner.py
+3
-1
@@ -8,7 +8,7 @@ from torch import nn
8
from vllm.attention import AttentionMetadata, get_attn_backend
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
ModelConfig, MultiModalConfig, ParallelConfig,
11
- SchedulerConfig)
11
+ PromptAdapterConfig, SchedulerConfig)
12
from vllm.logger import init_logger
13
from vllm.model_executor import SamplingMetadata
14
from vllm.model_executor.model_loader import get_model
@@ -81,6 +81,7 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
81
lora_config: Optional[LoRAConfig],
82
multimodal_config: Optional[MultiModalConfig],
83
kv_cache_dtype: Optional[str] = "auto",
84
+ prompt_adapter_config: Optional[PromptAdapterConfig] = None,
85
is_driver_worker: bool = False,
86
*args,
87
**kwargs,
@@ -94,6 +95,7 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
95
self.cache_config = cache_config
96
self.lora_config = lora_config
97
self.multimodal_config = multimodal_config
98
+ self.prompt_adapter_config = prompt_adapter_config
99
self.load_config = load_config
100
self.is_driver_worker = is_driver_worker
101
vllm/worker/cpu_worker.py
+4
-1
@@ -7,7 +7,7 @@ import torch.distributed
7
from vllm.attention import get_attn_backend
8
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
9
ModelConfig, MultiModalConfig, ParallelConfig,
10
- SchedulerConfig)
10
+ PromptAdapterConfig, SchedulerConfig)
11
from vllm.distributed import (ensure_model_parallel_initialized,
12
init_distributed_environment)
13
from vllm.logger import init_logger
@@ -133,6 +133,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
133
lora_config: Optional[LoRAConfig] = None,
134
multimodal_config: Optional[MultiModalConfig] = None,
135
kv_cache_dtype: Optional[str] = "auto",
136
+ prompt_adapter_config: Optional[PromptAdapterConfig] = None,
137
is_driver_worker: bool = False,
138
) -> None:
139
self.model_config = model_config
@@ -145,6 +146,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
146
self.rank = rank
147
self.distributed_init_method = distributed_init_method
148
self.lora_config = lora_config
149
+ self.prompt_adapter_config = prompt_adapter_config
150
self.multimodal_config = multimodal_config
151
self.is_driver_worker = is_driver_worker
152
if self.is_driver_worker:
@@ -167,6 +169,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
169
lora_config=self.lora_config,
170
multimodal_config=self.multimodal_config,
171
kv_cache_dtype=kv_cache_dtype,
172
+ prompt_adapter_config=self.prompt_adapter_config,
173
is_driver_worker=is_driver_worker)
174
# Uninitialized cache engine. Will be initialized by
175
# initialize_cache.
vllm/worker/embedding_model_runner.py
+10
-1
@@ -5,7 +5,7 @@ import torch
5
6
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7
ModelConfig, MultiModalConfig, ParallelConfig,
8
- SchedulerConfig)
8
+ PromptAdapterConfig, SchedulerConfig)
9
from vllm.logger import init_logger
10
from vllm.model_executor.pooling_metadata import PoolingMetadata
11
from vllm.pooling_params import PoolingParams
@@ -40,6 +40,7 @@ class EmbeddingModelRunner(
40
lora_config: Optional[LoRAConfig],
41
kv_cache_dtype: Optional[str] = "auto",
42
is_driver_worker: bool = False,
43
+ prompt_adapter_config: Optional[PromptAdapterConfig] = None,
44
multimodal_config: Optional[MultiModalConfig] = None,
45
):
46
super().__init__(model_config,
@@ -51,6 +52,7 @@ class EmbeddingModelRunner(
52
lora_config=lora_config,
53
kv_cache_dtype=kv_cache_dtype,
54
is_driver_worker=is_driver_worker,
55
+ prompt_adapter_config=prompt_adapter_config,
56
multimodal_config=multimodal_config)
57
58
@torch.inference_mode()
@@ -71,6 +73,13 @@ class EmbeddingModelRunner(
73
self.set_active_loras(model_input.lora_requests,
74
model_input.lora_mapping)
75
76
+ if self.prompt_adapter_config:
77
+ assert model_input.prompt_adapter_requests is not None
78
+ assert model_input.prompt_adapter_mapping is not None
79
+ self.set_active_prompt_adapters(
80
+ model_input.prompt_adapter_requests,
81
+ model_input.prompt_adapter_mapping)
82
+
83
# Currently cuda graph is only supported by the decode phase.
84
assert model_input.attn_metadata is not None
85
prefill_meta = model_input.attn_metadata.prefill_metadata
vllm/worker/model_runner.py
+118
-20
@@ -25,7 +25,7 @@ except ImportError:
25
from vllm.attention import AttentionMetadata, get_attn_backend
26
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
27
ModelConfig, MultiModalConfig, ParallelConfig,
28
- SchedulerConfig)
28
+ PromptAdapterConfig, SchedulerConfig)
29
from vllm.distributed import get_pp_group
30
from vllm.distributed.parallel_state import graph_capture
31
from vllm.inputs import INPUT_REGISTRY
@@ -40,6 +40,10 @@ from vllm.model_executor.models.interfaces import (supports_lora,
40
supports_vision)
41
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensors,
42
MultiModalInputs)
43
+from vllm.prompt_adapter.layers import PromptAdapterMapping
44
+from vllm.prompt_adapter.request import PromptAdapterRequest
45
+from vllm.prompt_adapter.worker_manager import (
46
+ LRUCacheWorkerPromptAdapterManager)
47
from vllm.sampling_params import SamplingParams
48
from vllm.sequence import (IntermediateTensors, SamplerOutput,
49
SequenceGroupMetadata)
@@ -85,6 +89,8 @@ class ModelInputForGPU(ModelRunnerInputBase):
89
lora_mapping: Optional["LoRAMapping"] = None
90
lora_requests: Optional[Set[LoRARequest]] = None
91
attn_metadata: Optional["AttentionMetadata"] = None
92
+ prompt_adapter_mapping: Optional[PromptAdapterMapping] = None
93
+ prompt_adapter_requests: Optional[Set[PromptAdapterRequest]] = None
94
multi_modal_kwargs: Optional[Mapping[str, BatchedTensors]] = None
95
request_ids_to_seq_ids: Optional[Dict[str, List[int]]] = None
96
finished_requests_ids: Optional[List[str]] = None
@@ -97,6 +103,8 @@ class ModelInputForGPU(ModelRunnerInputBase):
103
"lora_requests": self.lora_requests,
104
"lora_mapping": self.lora_mapping,
105
"multi_modal_kwargs": self.multi_modal_kwargs,
106
+ "prompt_adapter_mapping": self.prompt_adapter_mapping,
107
+ "prompt_adapter_requests": self.prompt_adapter_requests,
108
"virtual_engine": self.virtual_engine,
109
"request_ids_to_seq_ids": self.request_ids_to_seq_ids,
110
"finished_requests_ids": self.finished_requests_ids,
@@ -133,6 +141,8 @@ class ModelInputForGPUWithSamplingMetadata(ModelInputForGPU):
141
"lora_requests": self.lora_requests,
142
"lora_mapping": self.lora_mapping,
143
"multi_modal_kwargs": self.multi_modal_kwargs,
144
+ "prompt_adapter_mapping": self.prompt_adapter_mapping,
145
+ "prompt_adapter_requests": self.prompt_adapter_requests,
146
"virtual_engine": self.virtual_engine,
147
"request_ids_to_seq_ids": self.request_ids_to_seq_ids,
148
"finished_requests_ids": self.finished_requests_ids,
@@ -172,6 +182,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
182
lora_config: Optional[LoRAConfig],
183
kv_cache_dtype: Optional[str] = "auto",
184
is_driver_worker: bool = False,
185
+ prompt_adapter_config: Optional[PromptAdapterConfig] = None,
186
multimodal_config: Optional[MultiModalConfig] = None,
187
return_hidden_states: bool = False,
188
):
@@ -183,6 +194,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
194
self.lora_config = lora_config
195
self.load_config = load_config
196
self.is_driver_worker = is_driver_worker
197
+ self.prompt_adapter_config = prompt_adapter_config
198
self.multimodal_config = multimodal_config
199
self.return_hidden_states = return_hidden_states
200
@@ -232,6 +244,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
244
self.model: nn.Module # Set after load_model
245
# Set after load_model.
246
self.lora_manager: Optional[LRUCacheWorkerLoRAManager] = None
247
+ self.prompt_adapter_manager: LRUCacheWorkerPromptAdapterManager = None
248
249
self.flashinfer_decode_workspace_buffer = None
250
self.flashinfer_decode_wrapper = None
@@ -240,16 +253,14 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
253
254
def load_model(self) -> None:
255
with CudaMemoryProfiler() as m:
243
- self.model = get_model(
244
- model_config=self.model_config,
245
- device_config=self.device_config,
246
- load_config=self.load_config,
247
- lora_config=self.lora_config,
248
- multimodal_config=self.multimodal_config,
249
- parallel_config=self.parallel_config,
250
- scheduler_config=self.scheduler_config,
251
- cache_config=self.cache_config,
252
- )
256
+ self.model = get_model(model_config=self.model_config,
257
+ device_config=self.device_config,
258
+ load_config=self.load_config,
259
+ lora_config=self.lora_config,
260
+ multimodal_config=self.multimodal_config,
261
+ parallel_config=self.parallel_config,
262
+ scheduler_config=self.scheduler_config,
263
+ cache_config=self.cache_config)
264
265
self.model_memory_usage = m.consumed_memory
266
logger.info("Loading model weights took %.4f GB",
@@ -274,6 +285,15 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
285
)
286
self.model = self.lora_manager.create_lora_manager(self.model)
287
288
+ if self.prompt_adapter_config:
289
+ self.prompt_adapter_manager = LRUCacheWorkerPromptAdapterManager(
290
+ self.scheduler_config.max_num_seqs,
291
+ self.scheduler_config.max_num_batched_tokens, self.device,
292
+ self.prompt_adapter_config)
293
+ self.model = (
294
+ self.prompt_adapter_manager.create_prompt_adapter_manager(
295
+ self.model))
296
+
297
if self.kv_cache_dtype == "fp8" and is_hip():
298
# Currently only ROCm accepts kv-cache scaling factors
299
# via quantization_param_path and this will be deprecated
@@ -354,6 +374,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
374
lora_index_mapping: List[int] = []
375
lora_prompt_mapping: List[int] = []
376
lora_requests: Set[LoRARequest] = set()
377
+ prompt_adapter_index_mapping: List[int] = []
378
+ prompt_adapter_prompt_mapping: List[int] = []
379
+ prompt_adapter_requests: Set[PromptAdapterRequest] = set()
380
381
seq_lens: List[int] = []
382
prefill_seq_lens: List[int] = []
@@ -504,6 +527,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
527
input_tokens.extend(tokens)
528
input_positions.extend(list(range(context_len, seq_len)))
529
lora_id = seq_group_metadata.lora_int_id
530
+ prompt_adapter_id = seq_group_metadata.prompt_adapter_id
531
532
if is_prompt:
533
assert len(seq_ids) == 1
@@ -534,6 +558,21 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
558
mm_kwargs = self.multi_modal_input_mapper(mm_data)
559
multi_modal_inputs_list.append(mm_kwargs)
560
561
+ if prompt_adapter_id > 0 and is_prompt:
562
+ prompt_adapter_requests.add(
563
+ seq_group_metadata.prompt_adapter_request)
564
+
565
+ num_tokens = seq_group_metadata.\
566
+ prompt_adapter_num_virtual_tokens
567
+ pm = [prompt_adapter_id
568
+ ] * num_tokens + [0] * (query_len - num_tokens)
569
+ prompt_adapter_index_mapping += pm
570
+ prompt_adapter_prompt_mapping.extend(
571
+ [prompt_adapter_id] *
572
+ (query_len if seq_group_metadata.sampling_params
573
+ and seq_group_metadata.sampling_params.prompt_logprobs
574
+ else 1))
575
+
576
is_profile_run = _is_block_tables_empty(
577
seq_group_metadata.block_tables)
578
if is_profile_run:
@@ -618,12 +657,11 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
657
seq_lens.append(1)
658
block_tables.append([])
659
lora_index_mapping.append(0)
621
-
660
+ prompt_adapter_index_mapping.append(0)
661
if self.attn_backend.get_name() == "flashinfer":
662
last_paged_kv_indptr = paged_kv_indptr[-1]
663
paged_kv_indptr.append(last_paged_kv_indptr)
664
paged_kv_last_page_len.append(0)
626
-
665
batch_size = graph_batch_size
666
num_decode_tokens = batch_size
667
@@ -759,6 +797,14 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
797
else:
798
lora_mapping = None
799
800
+ if self.prompt_adapter_config:
801
+ prompt_adapter_mapping = PromptAdapterMapping(
802
+ prompt_adapter_index_mapping,
803
+ prompt_adapter_prompt_mapping,
804
+ )
805
+ else:
806
+ prompt_adapter_mapping = None
807
+
808
multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list,
809
device=self.device)
810
request_ids_to_seq_ids = {
@@ -776,7 +822,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
822
lora_requests=lora_requests,
823
multi_modal_kwargs=multi_modal_kwargs,
824
request_ids_to_seq_ids=request_ids_to_seq_ids,
779
- finished_requests_ids=finished_requests_ids)
825
+ finished_requests_ids=finished_requests_ids,
826
+ prompt_adapter_mapping=prompt_adapter_mapping,
827
+ prompt_adapter_requests=prompt_adapter_requests,
828
+ )
829
830
@torch.inference_mode()
831
def profile_run(self) -> None:
@@ -878,33 +927,67 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
927
def remove_all_loras(self):
928
if not self.lora_manager:
929
raise RuntimeError("LoRA is not enabled.")
881
- self.lora_manager.remove_all_loras()
930
+ self.lora_manager.remove_all_adapters()
931
932
def set_active_loras(self, lora_requests: Set[LoRARequest],
933
lora_mapping: LoRAMapping) -> None:
934
if not self.lora_manager:
935
raise RuntimeError("LoRA is not enabled.")
887
- self.lora_manager.set_active_loras(lora_requests, lora_mapping)
936
+ self.lora_manager.set_active_adapters(lora_requests, lora_mapping)
937
938
def add_lora(self, lora_request: LoRARequest) -> bool:
939
if not self.lora_manager:
940
raise RuntimeError("LoRA is not enabled.")
892
- return self.lora_manager.add_lora(lora_request)
941
+ return self.lora_manager.add_adapter(lora_request)
942
943
def remove_lora(self, lora_id: int) -> bool:
944
if not self.lora_manager:
945
raise RuntimeError("LoRA is not enabled.")
897
- return self.lora_manager.remove_lora(lora_id)
946
+ return self.lora_manager.remove_adapter(lora_id)
947
948
def pin_lora(self, lora_id: int) -> bool:
949
if not self.lora_manager:
950
raise RuntimeError("LoRA is not enabled.")
902
- return self.lora_manager.pin_lora(lora_id)
951
+ return self.lora_manager.pin_adapter(lora_id)
952
953
def list_loras(self) -> Set[int]:
954
if not self.lora_manager:
955
raise RuntimeError("LoRA is not enabled.")
907
- return self.lora_manager.list_loras()
956
+ return self.lora_manager.list_adapters()
957
+
958
+ def remove_all_prompt_adapters(self):
959
+ if not self.prompt_adapter_manager:
960
+ raise RuntimeError("PromptAdapter is not enabled.")
961
+ self.prompt_adapter_manager.remove_all_adapters()
962
+
963
+ def set_active_prompt_adapters(
964
+ self, prompt_adapter_requests: Set[PromptAdapterRequest],
965
+ prompt_adapter_mapping: PromptAdapterMapping) -> None:
966
+ if not self.prompt_adapter_manager:
967
+ raise RuntimeError("PromptAdapter is not enabled.")
968
+ self.prompt_adapter_manager.set_active_adapters(
969
+ prompt_adapter_requests, prompt_adapter_mapping)
970
+
971
+ def add_prompt_adapter(
972
+ self, prompt_adapter_request: PromptAdapterRequest) -> bool:
973
+ if not self.prompt_adapter_manager:
974
+ raise RuntimeError("PromptAdapter is not enabled.")
975
+ return self.prompt_adapter_manager.add_adapter(prompt_adapter_request)
976
+
977
+ def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
978
+ if not self.prompt_adapter_manager:
979
+ raise RuntimeError("PromptAdapter is not enabled.")
980
+ return self.prompt_adapter_manager.remove_adapter(prompt_adapter_id)
981
+
982
+ def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
983
+ if not self.prompt_adapter_manager:
984
+ raise RuntimeError("PromptAdapter is not enabled.")
985
+ return self.prompt_adapter_manager.pin_adapter(prompt_adapter_id)
986
+
987
+ def list_prompt_adapters(self) -> Set[int]:
988
+ if not self.prompt_adapter_manager:
989
+ raise RuntimeError("PromptAdapter is not enabled.")
990
+ return self.prompt_adapter_manager.list_adapters()
991
992
@torch.inference_mode()
993
def capture_model(self, kv_caches: List[List[torch.Tensor]]) -> None:
@@ -1063,6 +1146,14 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1146
)
1147
self.set_active_loras(set(), lora_mapping)
1148
1149
+ if self.prompt_adapter_config:
1150
+ prompt_adapter_mapping = PromptAdapterMapping(
1151
+ [-1] * batch_size,
1152
+ [-1] * batch_size,
1153
+ )
1154
+ self.set_active_prompt_adapters(
1155
+ set(), prompt_adapter_mapping)
1156
+
1157
graph_runner = CUDAGraphRunner(
1158
self.model, self.attn_backend.get_name())
1159
@@ -1189,6 +1280,13 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1280
self.set_active_loras(model_input.lora_requests,
1281
model_input.lora_mapping)
1282
1283
+ if self.prompt_adapter_config:
1284
+ assert model_input.prompt_adapter_requests is not None
1285
+ assert model_input.prompt_adapter_mapping is not None
1286
+ self.set_active_prompt_adapters(
1287
+ model_input.prompt_adapter_requests,
1288
+ model_input.prompt_adapter_mapping)
1289
+
1290
if self.attn_backend.get_name() == "flashinfer":
1291
assert model_input.attn_metadata is not None
1292
assert model_input.input_tokens is not None
vllm/worker/worker.py
+19
-1
@@ -8,7 +8,8 @@ import torch.distributed
8
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
ModelConfig, MultiModalConfig, ParallelConfig,
11
- SchedulerConfig, SpeculativeConfig)
11
+ PromptAdapterConfig, SchedulerConfig,
12
+ SpeculativeConfig)
13
from vllm.distributed import (ensure_model_parallel_initialized,
14
init_distributed_environment,
15
set_custom_all_reduce)
@@ -16,6 +17,7 @@ from vllm.lora.request import LoRARequest
17
from vllm.model_executor import set_random_seed
18
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
19
from vllm.platforms import current_platform
20
+from vllm.prompt_adapter.request import PromptAdapterRequest
21
from vllm.sequence import ExecuteModelRequest
22
from vllm.worker.cache_engine import CacheEngine
23
from vllm.worker.embedding_model_runner import EmbeddingModelRunner
@@ -45,6 +47,7 @@ class Worker(LocalOrDistributedWorkerBase):
47
lora_config: Optional[LoRAConfig] = None,
48
multimodal_config: Optional[MultiModalConfig] = None,
49
speculative_config: Optional[SpeculativeConfig] = None,
50
+ prompt_adapter_config: Optional[PromptAdapterConfig] = None,
51
is_driver_worker: bool = False,
52
model_runner_cls: Optional[Type[GPUModelRunnerBase]] = None,
53
) -> None:
@@ -59,6 +62,7 @@ class Worker(LocalOrDistributedWorkerBase):
62
self.distributed_init_method = distributed_init_method
63
self.lora_config = lora_config
64
self.load_config = load_config
65
+ self.prompt_adapter_config = prompt_adapter_config
66
self.is_driver_worker = is_driver_worker
67
if parallel_config and is_driver_worker:
68
assert rank % parallel_config.tensor_parallel_size == 0, \
@@ -92,6 +96,7 @@ class Worker(LocalOrDistributedWorkerBase):
96
lora_config=self.lora_config,
97
kv_cache_dtype=self.cache_config.cache_dtype,
98
is_driver_worker=is_driver_worker,
99
+ prompt_adapter_config=prompt_adapter_config,
100
multimodal_config=multimodal_config,
101
**speculative_args,
102
)
@@ -296,6 +301,19 @@ class Worker(LocalOrDistributedWorkerBase):
301
def list_loras(self) -> Set[int]:
302
return self.model_runner.list_loras()
303
304
+ def add_prompt_adapter(
305
+ self, prompt_adapter_request: PromptAdapterRequest) -> bool:
306
+ return self.model_runner.add_prompt_adapter(prompt_adapter_request)
307
+
308
+ def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
309
+ return self.model_runner.remove_lora(prompt_adapter_id)
310
+
311
+ def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
312
+ return self.model_runner.pin_prompt_adapter(prompt_adapter_id)
313
+
314
+ def list_prompt_adapters(self) -> Set[int]:
315
+ return self.model_runner.list_prompt_adapters()
316
+
317
@property
318
def max_model_len(self) -> int:
319
return self.model_config.max_model_len
vllm/worker/xpu_model_runner.py
+3
-1
@@ -8,7 +8,7 @@ import torch.nn as nn
8
from vllm.attention import get_attn_backend
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
ModelConfig, MultiModalConfig, ParallelConfig,
11
- SchedulerConfig)
11
+ PromptAdapterConfig, SchedulerConfig)
12
from vllm.distributed import broadcast_tensor_dict
13
from vllm.inputs import INPUT_REGISTRY
14
from vllm.logger import init_logger
@@ -88,6 +88,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
88
lora_config: Optional[LoRAConfig],
89
multimodal_config: Optional[MultiModalConfig],
90
kv_cache_dtype: Optional[str] = "auto",
91
+ prompt_adapter_config: Optional[PromptAdapterConfig] = None,
92
is_driver_worker: bool = False,
93
*args,
94
**kwargs,
@@ -98,6 +99,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
99
self.lora_config = lora_config
100
self.load_config = load_config
101
self.cache_config = cache_config
102
+ self.prompt_adapter_config = prompt_adapter_config
103
self.multimodal_config = multimodal_config
104
self.is_driver_worker = is_driver_worker
105
vllm/worker/xpu_worker.py
+4
-1
@@ -10,7 +10,8 @@ import torch.distributed
10
11
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12
ModelConfig, MultiModalConfig, ParallelConfig,
13
- SchedulerConfig, SpeculativeConfig)
13
+ PromptAdapterConfig, SchedulerConfig,
14
+ SpeculativeConfig)
15
from vllm.distributed import (ensure_model_parallel_initialized,
16
init_distributed_environment)
17
from vllm.logger import init_logger
@@ -47,6 +48,7 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
48
lora_config: Optional[LoRAConfig] = None,
49
multimodal_config: Optional[MultiModalConfig] = None,
50
speculative_config: Optional[SpeculativeConfig] = None,
51
+ prompt_adapter_config: Optional[PromptAdapterConfig] = None,
52
is_driver_worker: bool = False,
53
) -> None:
54
assert device_config.device_type == "xpu"
@@ -63,6 +65,7 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
65
self.rank = rank
66
self.distributed_init_method = distributed_init_method
67
self.lora_config = lora_config
68
+ self.prompt_adapter_config = prompt_adapter_config
69
self.is_driver_worker = is_driver_worker
70
if self.is_driver_worker:
71
assert self.rank == 0, "The driver worker must have rank 0."