A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Frontend] enable passing multiple LoRA adapters at once to generate() (#5300)
Matthew Goldey committed
Jun 6, 2024 at 16:48 UTC
828da0d44e9124d949909477d6018fc08469a31e
2 files changed
+91
-17
tests/entrypoints/test_llm_generate_multiple_loras.py
new
+69
@@ -0,0 +1,69 @@
1
+import weakref
2
+
3
+import pytest
4
+# downloading lora to test lora requests
5
+from huggingface_hub import snapshot_download
6
+
7
+from vllm import LLM
8
+from vllm.lora.request import LoRARequest
9
+
10
+from ..conftest import cleanup
11
+
12
+MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
13
+
14
+PROMPTS = [
15
+ "Hello, my name is",
16
+ "The president of the United States is",
17
+ "The capital of France is",
18
+ "The future of AI is",
19
+]
20
+
21
+LORA_NAME = "typeof/zephyr-7b-beta-lora"
22
+
23
+pytestmark = pytest.mark.llm
24
+
25
+
26
+@pytest.fixture(scope="module")
27
+def llm():
28
+ # pytest caches the fixture so we use weakref.proxy to
29
+ # enable garbage collection
30
+ llm = LLM(model=MODEL_NAME,
31
+ tensor_parallel_size=1,
32
+ max_model_len=8192,
33
+ enable_lora=True,
34
+ max_loras=4,
35
+ max_lora_rank=64,
36
+ max_num_seqs=128,
37
+ enforce_eager=True)
38
+
39
+ with llm.deprecate_legacy_api():
40
+ yield weakref.proxy(llm)
41
+
42
+ del llm
43
+
44
+ cleanup()
45
+
46
+
47
+@pytest.fixture(scope="session")
48
+def zephyr_lora_files():
49
+ return snapshot_download(repo_id=LORA_NAME)
50
+
51
+
52
+@pytest.mark.skip_global_cleanup
53
+def test_multiple_lora_requests(llm: LLM, zephyr_lora_files):
54
+ lora_request = [
55
+ LoRARequest(LORA_NAME, idx + 1, zephyr_lora_files)
56
+ for idx in range(len(PROMPTS))
57
+ ]
58
+ # Multiple SamplingParams should be matched with each prompt
59
+ outputs = llm.generate(PROMPTS, lora_request=lora_request)
60
+ assert len(PROMPTS) == len(outputs)
61
+
62
+ # Exception raised, if the size of params does not match the size of prompts
63
+ with pytest.raises(ValueError):
64
+ outputs = llm.generate(PROMPTS, lora_request=lora_request[:1])
65
+
66
+ # Single LoRARequest should be applied to every prompt
67
+ single_lora_request = lora_request[0]
68
+ outputs = llm.generate(PROMPTS, lora_request=single_lora_request)
69
+ assert len(PROMPTS) == len(outputs)
vllm/entrypoints/llm.py
+22
-17
@@ -170,7 +170,7 @@ class LLM:
170
List[SamplingParams]]] = None,
171
prompt_token_ids: Optional[List[int]] = None,
172
use_tqdm: bool = True,
173
- lora_request: Optional[LoRARequest] = None,
173
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
174
) -> List[RequestOutput]:
175
...
176
@@ -182,7 +182,7 @@ class LLM:
182
List[SamplingParams]]] = None,
183
prompt_token_ids: Optional[List[List[int]]] = None,
184
use_tqdm: bool = True,
185
- lora_request: Optional[LoRARequest] = None,
185
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
186
) -> List[RequestOutput]:
187
...
188
@@ -195,7 +195,7 @@ class LLM:
195
*,
196
prompt_token_ids: List[int],
197
use_tqdm: bool = True,
198
- lora_request: Optional[LoRARequest] = None,
198
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
199
) -> List[RequestOutput]:
200
...
201
@@ -208,7 +208,7 @@ class LLM:
208
*,
209
prompt_token_ids: List[List[int]],
210
use_tqdm: bool = True,
211
- lora_request: Optional[LoRARequest] = None,
211
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
212
) -> List[RequestOutput]:
213
...
214
@@ -219,7 +219,7 @@ class LLM:
219
sampling_params: None,
220
prompt_token_ids: Union[List[int], List[List[int]]],
221
use_tqdm: bool = True,
222
- lora_request: Optional[LoRARequest] = None,
222
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
223
) -> List[RequestOutput]:
224
...
225
@@ -232,7 +232,7 @@ class LLM:
232
sampling_params: Optional[Union[SamplingParams,
233
Sequence[SamplingParams]]] = None,
234
use_tqdm: bool = True,
235
- lora_request: Optional[LoRARequest] = None,
235
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
236
) -> List[RequestOutput]:
237
...
238
@@ -249,7 +249,7 @@ class LLM:
249
Sequence[SamplingParams]]] = None,
250
prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
251
use_tqdm: bool = True,
252
- lora_request: Optional[LoRARequest] = None,
252
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
253
) -> List[RequestOutput]:
254
"""Generates the completions for the input prompts.
255
@@ -312,7 +312,7 @@ class LLM:
312
Sequence[PoolingParams]]] = None,
313
prompt_token_ids: Optional[List[int]] = None,
314
use_tqdm: bool = True,
315
- lora_request: Optional[LoRARequest] = None,
315
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
316
) -> List[EmbeddingRequestOutput]:
317
...
318
@@ -324,7 +324,7 @@ class LLM:
324
Sequence[PoolingParams]]] = None,
325
prompt_token_ids: Optional[List[List[int]]] = None,
326
use_tqdm: bool = True,
327
- lora_request: Optional[LoRARequest] = None,
327
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
328
) -> List[EmbeddingRequestOutput]:
329
...
330
@@ -337,7 +337,7 @@ class LLM:
337
*,
338
prompt_token_ids: List[int],
339
use_tqdm: bool = True,
340
- lora_request: Optional[LoRARequest] = None,
340
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
341
) -> List[EmbeddingRequestOutput]:
342
...
343
@@ -350,7 +350,7 @@ class LLM:
350
*,
351
prompt_token_ids: List[List[int]],
352
use_tqdm: bool = True,
353
- lora_request: Optional[LoRARequest] = None,
353
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
354
) -> List[EmbeddingRequestOutput]:
355
...
356
@@ -361,7 +361,7 @@ class LLM:
361
pooling_params: None,
362
prompt_token_ids: Union[List[int], List[List[int]]],
363
use_tqdm: bool = True,
364
- lora_request: Optional[LoRARequest] = None,
364
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
365
) -> List[EmbeddingRequestOutput]:
366
...
367
@@ -374,7 +374,7 @@ class LLM:
374
pooling_params: Optional[Union[PoolingParams,
375
Sequence[PoolingParams]]] = None,
376
use_tqdm: bool = True,
377
- lora_request: Optional[LoRARequest] = None,
377
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
378
) -> List[EmbeddingRequestOutput]:
379
...
380
@@ -391,7 +391,7 @@ class LLM:
391
Sequence[PoolingParams]]] = None,
392
prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
393
use_tqdm: bool = True,
394
- lora_request: Optional[LoRARequest] = None,
394
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
395
) -> List[EmbeddingRequestOutput]:
396
"""Generates the completions for the input prompts.
397
@@ -498,7 +498,7 @@ class LLM:
498
inputs: Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
499
params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
500
Sequence[PoolingParams]],
501
- lora_request: Optional[LoRARequest],
501
+ lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
502
) -> None:
503
if isinstance(inputs, (str, dict)):
504
# Convert a single prompt to a list.
@@ -509,20 +509,25 @@ class LLM:
509
if isinstance(params, list) and len(params) != num_requests:
510
raise ValueError("The lengths of prompts and params "
511
"must be the same.")
512
+ if isinstance(lora_request,
513
+ list) and len(lora_request) != num_requests:
514
+ raise ValueError("The lengths of prompts and lora_request "
515
+ "must be the same.")
516
517
# Add requests to the engine.
518
for i, request_inputs in enumerate(inputs):
519
self._add_request(
520
request_inputs,
521
params[i] if isinstance(params, Sequence) else params,
518
- lora_request=lora_request,
522
+ lora_request=lora_request[i] if isinstance(
523
+ lora_request, Sequence) else lora_request,
524
)
525
526
def _add_request(
527
self,
528
inputs: PromptInputs,
529
params: Union[SamplingParams, PoolingParams],
525
- lora_request: Optional[LoRARequest] = None,
530
+ lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
531
) -> None:
532
request_id = str(next(self.request_counter))
533
self.llm_engine.add_request(request_id,