@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 828da0d44e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Frontend] enable passing multiple LoRA adapters at once to generate() (#5300)

Matthew Goldey committed Jun 6, 2024 at 16:48 UTC 828da0d44e9124d949909477d6018fc08469a31e
2 files changed +91 -17
tests/entrypoints/test_llm_generate_multiple_loras.py new
+69
@@ -0,0 +1,69 @@
1 +import weakref
2 +
3 +import pytest
4 +# downloading lora to test lora requests
5 +from huggingface_hub import snapshot_download
6 +
7 +from vllm import LLM
8 +from vllm.lora.request import LoRARequest
9 +
10 +from ..conftest import cleanup
11 +
12 +MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
13 +
14 +PROMPTS = [
15 + "Hello, my name is",
16 + "The president of the United States is",
17 + "The capital of France is",
18 + "The future of AI is",
19 +]
20 +
21 +LORA_NAME = "typeof/zephyr-7b-beta-lora"
22 +
23 +pytestmark = pytest.mark.llm
24 +
25 +
26 +@pytest.fixture(scope="module")
27 +def llm():
28 + # pytest caches the fixture so we use weakref.proxy to
29 + # enable garbage collection
30 + llm = LLM(model=MODEL_NAME,
31 + tensor_parallel_size=1,
32 + max_model_len=8192,
33 + enable_lora=True,
34 + max_loras=4,
35 + max_lora_rank=64,
36 + max_num_seqs=128,
37 + enforce_eager=True)
38 +
39 + with llm.deprecate_legacy_api():
40 + yield weakref.proxy(llm)
41 +
42 + del llm
43 +
44 + cleanup()
45 +
46 +
47 +@pytest.fixture(scope="session")
48 +def zephyr_lora_files():
49 + return snapshot_download(repo_id=LORA_NAME)
50 +
51 +
52 +@pytest.mark.skip_global_cleanup
53 +def test_multiple_lora_requests(llm: LLM, zephyr_lora_files):
54 + lora_request = [
55 + LoRARequest(LORA_NAME, idx + 1, zephyr_lora_files)
56 + for idx in range(len(PROMPTS))
57 + ]
58 + # Multiple SamplingParams should be matched with each prompt
59 + outputs = llm.generate(PROMPTS, lora_request=lora_request)
60 + assert len(PROMPTS) == len(outputs)
61 +
62 + # Exception raised, if the size of params does not match the size of prompts
63 + with pytest.raises(ValueError):
64 + outputs = llm.generate(PROMPTS, lora_request=lora_request[:1])
65 +
66 + # Single LoRARequest should be applied to every prompt
67 + single_lora_request = lora_request[0]
68 + outputs = llm.generate(PROMPTS, lora_request=single_lora_request)
69 + assert len(PROMPTS) == len(outputs)
vllm/entrypoints/llm.py
+22 -17
@@ -170,7 +170,7 @@ class LLM:
170 List[SamplingParams]]] = None,
171 prompt_token_ids: Optional[List[int]] = None,
172 use_tqdm: bool = True,
173 - lora_request: Optional[LoRARequest] = None,
173 + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
174 ) -> List[RequestOutput]:
175 ...
176
@@ -182,7 +182,7 @@ class LLM:
182 List[SamplingParams]]] = None,
183 prompt_token_ids: Optional[List[List[int]]] = None,
184 use_tqdm: bool = True,
185 - lora_request: Optional[LoRARequest] = None,
185 + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
186 ) -> List[RequestOutput]:
187 ...
188
@@ -195,7 +195,7 @@ class LLM:
195 *,
196 prompt_token_ids: List[int],
197 use_tqdm: bool = True,
198 - lora_request: Optional[LoRARequest] = None,
198 + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
199 ) -> List[RequestOutput]:
200 ...
201
@@ -208,7 +208,7 @@ class LLM:
208 *,
209 prompt_token_ids: List[List[int]],
210 use_tqdm: bool = True,
211 - lora_request: Optional[LoRARequest] = None,
211 + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
212 ) -> List[RequestOutput]:
213 ...
214
@@ -219,7 +219,7 @@ class LLM:
219 sampling_params: None,
220 prompt_token_ids: Union[List[int], List[List[int]]],
221 use_tqdm: bool = True,
222 - lora_request: Optional[LoRARequest] = None,
222 + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
223 ) -> List[RequestOutput]:
224 ...
225
@@ -232,7 +232,7 @@ class LLM:
232 sampling_params: Optional[Union[SamplingParams,
233 Sequence[SamplingParams]]] = None,
234 use_tqdm: bool = True,
235 - lora_request: Optional[LoRARequest] = None,
235 + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
236 ) -> List[RequestOutput]:
237 ...
238
@@ -249,7 +249,7 @@ class LLM:
249 Sequence[SamplingParams]]] = None,
250 prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
251 use_tqdm: bool = True,
252 - lora_request: Optional[LoRARequest] = None,
252 + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
253 ) -> List[RequestOutput]:
254 """Generates the completions for the input prompts.
255
@@ -312,7 +312,7 @@ class LLM:
312 Sequence[PoolingParams]]] = None,
313 prompt_token_ids: Optional[List[int]] = None,
314 use_tqdm: bool = True,
315 - lora_request: Optional[LoRARequest] = None,
315 + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
316 ) -> List[EmbeddingRequestOutput]:
317 ...
318
@@ -324,7 +324,7 @@ class LLM:
324 Sequence[PoolingParams]]] = None,
325 prompt_token_ids: Optional[List[List[int]]] = None,
326 use_tqdm: bool = True,
327 - lora_request: Optional[LoRARequest] = None,
327 + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
328 ) -> List[EmbeddingRequestOutput]:
329 ...
330
@@ -337,7 +337,7 @@ class LLM:
337 *,
338 prompt_token_ids: List[int],
339 use_tqdm: bool = True,
340 - lora_request: Optional[LoRARequest] = None,
340 + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
341 ) -> List[EmbeddingRequestOutput]:
342 ...
343
@@ -350,7 +350,7 @@ class LLM:
350 *,
351 prompt_token_ids: List[List[int]],
352 use_tqdm: bool = True,
353 - lora_request: Optional[LoRARequest] = None,
353 + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
354 ) -> List[EmbeddingRequestOutput]:
355 ...
356
@@ -361,7 +361,7 @@ class LLM:
361 pooling_params: None,
362 prompt_token_ids: Union[List[int], List[List[int]]],
363 use_tqdm: bool = True,
364 - lora_request: Optional[LoRARequest] = None,
364 + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
365 ) -> List[EmbeddingRequestOutput]:
366 ...
367
@@ -374,7 +374,7 @@ class LLM:
374 pooling_params: Optional[Union[PoolingParams,
375 Sequence[PoolingParams]]] = None,
376 use_tqdm: bool = True,
377 - lora_request: Optional[LoRARequest] = None,
377 + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
378 ) -> List[EmbeddingRequestOutput]:
379 ...
380
@@ -391,7 +391,7 @@ class LLM:
391 Sequence[PoolingParams]]] = None,
392 prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
393 use_tqdm: bool = True,
394 - lora_request: Optional[LoRARequest] = None,
394 + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
395 ) -> List[EmbeddingRequestOutput]:
396 """Generates the completions for the input prompts.
397
@@ -498,7 +498,7 @@ class LLM:
498 inputs: Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
499 params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
500 Sequence[PoolingParams]],
501 - lora_request: Optional[LoRARequest],
501 + lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
502 ) -> None:
503 if isinstance(inputs, (str, dict)):
504 # Convert a single prompt to a list.
@@ -509,20 +509,25 @@ class LLM:
509 if isinstance(params, list) and len(params) != num_requests:
510 raise ValueError("The lengths of prompts and params "
511 "must be the same.")
512 + if isinstance(lora_request,
513 + list) and len(lora_request) != num_requests:
514 + raise ValueError("The lengths of prompts and lora_request "
515 + "must be the same.")
516
517 # Add requests to the engine.
518 for i, request_inputs in enumerate(inputs):
519 self._add_request(
520 request_inputs,
521 params[i] if isinstance(params, Sequence) else params,
518 - lora_request=lora_request,
522 + lora_request=lora_request[i] if isinstance(
523 + lora_request, Sequence) else lora_request,
524 )
525
526 def _add_request(
527 self,
528 inputs: PromptInputs,
529 params: Union[SamplingParams, PoolingParams],
525 - lora_request: Optional[LoRARequest] = None,
530 + lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
531 ) -> None:
532 request_id = str(next(self.request_counter))
533 self.llm_engine.add_request(request_id,