A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[misc] benchmark_throughput : Add LoRA (#11267)
Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Varun Sundar Rabindranath committed
Dec 19, 2024 at 02:43 UTC
98356735ac51e25877f1b63c5f0733df7cebe5f7
1 file changed
+89
-13
benchmarks/benchmark_throughput.py
+89
-13
@@ -4,7 +4,8 @@ import dataclasses
4
import json
5
import random
6
import time
7
-from typing import List, Optional
7
+from functools import cache
8
+from typing import Dict, List, Optional, Tuple
9
10
import torch
11
import uvloop
@@ -17,8 +18,11 @@ from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
18
from vllm.entrypoints.openai.api_server import (
19
build_async_engine_client_from_engine_args)
20
from vllm.inputs import TextPrompt
21
+from vllm.lora.request import LoRARequest
22
+from vllm.lora.utils import get_adapter_absolute_path
23
from vllm.multimodal import MultiModalDataDict
24
from vllm.sampling_params import BeamSearchParams
25
+from vllm.transformers_utils.tokenizer import AnyTokenizer, get_lora_tokenizer
26
from vllm.utils import FlexibleArgumentParser, merge_async_iterators
27
28
@@ -28,15 +32,17 @@ class SampleRequest:
32
33
Attributes:
34
prompt: The input text prompt for the model.
31
- multi_modal_data: Optional dictionary containing multi-modal data (e.g.
32
- images).
35
prompt_len: The length of the prompt in tokens.
36
expected_output_len: The expected length of the output in tokens.
37
+ multi_modal_data: Optional dictionary containing multi-modal data (e.g.
38
+ images).
39
+ lora_request: Optional LoRARequest specifying the LoRA to use.
40
"""
41
prompt: str
42
prompt_len: int
43
expected_output_len: int
44
multi_modal_data: Optional[MultiModalDataDict] = None
45
+ lora_request: Optional[LoRARequest] = None
46
47
48
def _get_prompt_for_image_model(question: str, *, model: str) -> str:
@@ -60,8 +66,30 @@ def _get_prompt_for_image_model(question: str, *, model: str) -> str:
66
raise ValueError(f"Unsupported model {model}")
67
68
69
+@cache
70
+def lora_path_on_disk(lora_path: str) -> str:
71
+ return get_adapter_absolute_path(lora_path)
72
+
73
+
74
+lora_tokenizer_cache: Dict[int, AnyTokenizer] = {}
75
+
76
+
77
+def get_random_lora_request(
78
+ args: argparse.Namespace
79
+) -> Tuple[LoRARequest, Optional[AnyTokenizer]]:
80
+ global lora_tokenizer_cache
81
+ lora_id = random.randint(1, args.max_loras)
82
+ lora_request = LoRARequest(lora_name=str(lora_id),
83
+ lora_int_id=lora_id,
84
+ lora_path=lora_path_on_disk(args.lora_path))
85
+ if lora_id not in lora_tokenizer_cache:
86
+ lora_tokenizer_cache[lora_id] = get_lora_tokenizer(lora_request)
87
+ return lora_request, lora_tokenizer_cache[lora_id]
88
+
89
+
90
def sample_requests(tokenizer: PreTrainedTokenizerBase,
91
args: argparse.Namespace) -> List[SampleRequest]:
92
+
93
dataset_path: str = args.dataset
94
num_requests: int = args.num_prompts
95
fixed_output_len: Optional[int] = args.output_len
@@ -79,7 +107,9 @@ def sample_requests(tokenizer: PreTrainedTokenizerBase,
107
108
# Filter out sequences that are too long or too short
109
filtered_dataset: List[SampleRequest] = []
82
- for data in dataset:
110
+ for data in tqdm(dataset,
111
+ total=len(filtered_dataset),
112
+ desc="sampling requests"):
113
if len(filtered_dataset) == num_requests:
114
break
115
@@ -102,9 +132,16 @@ def sample_requests(tokenizer: PreTrainedTokenizerBase,
132
continue
133
prompt = _get_prompt_for_image_model(question=prompt, model=model)
134
135
+ request_tokenizer = tokenizer
136
+ lora_request: Optional[LoRARequest] = None
137
+ if args.enable_lora:
138
+ lora_request, lora_tokenizer = get_random_lora_request(args)
139
+ if lora_tokenizer:
140
+ request_tokenizer = lora_tokenizer
141
+
142
# Tokenize the prompts and completions.
106
- prompt_token_ids = tokenizer(prompt).input_ids
107
- completion_token_ids = tokenizer(completion).input_ids
143
+ prompt_token_ids = request_tokenizer(prompt).input_ids
144
+ completion_token_ids = request_tokenizer(completion).input_ids
145
prompt_len = len(prompt_token_ids)
146
output_len = len(completion_token_ids
147
) if fixed_output_len is None else fixed_output_len
@@ -118,7 +155,8 @@ def sample_requests(tokenizer: PreTrainedTokenizerBase,
155
SampleRequest(prompt=prompt,
156
prompt_len=prompt_len,
157
expected_output_len=output_len,
121
- multi_modal_data=multi_modal_data))
158
+ multi_modal_data=multi_modal_data,
159
+ lora_request=lora_request))
160
161
return filtered_dataset
162
@@ -146,14 +184,21 @@ def run_vllm(
184
ignore_eos=True,
185
max_tokens=request.expected_output_len,
186
))
187
+ lora_requests: Optional[List[LoRARequest]] = None
188
+ if engine_args.enable_lora:
189
+ lora_requests = [request.lora_request for request in requests]
190
191
use_beam_search = False
192
193
if not use_beam_search:
194
start = time.perf_counter()
154
- llm.generate(prompts, sampling_params, use_tqdm=True)
195
+ llm.generate(prompts,
196
+ sampling_params,
197
+ lora_request=lora_requests,
198
+ use_tqdm=True)
199
end = time.perf_counter()
200
else:
201
+ assert lora_requests is None, "BeamSearch API does not support LoRA"
202
prompts = [request.prompt for request in requests]
203
# output_len should be the same for all requests.
204
output_len = requests[0][2]
@@ -185,6 +230,7 @@ async def run_vllm_async(
230
# Add the requests to the engine.
231
prompts: List[TextPrompt] = []
232
sampling_params: List[SamplingParams] = []
233
+ lora_requests: List[Optional[LoRARequest]] = []
234
for request in requests:
235
prompts.append(
236
TextPrompt(prompt=request.prompt,
@@ -197,11 +243,16 @@ async def run_vllm_async(
243
ignore_eos=True,
244
max_tokens=request.expected_output_len,
245
))
246
+ lora_requests.append(request.lora_request)
247
248
generators = []
249
start = time.perf_counter()
203
- for i, (prompt, sp) in enumerate(zip(prompts, sampling_params)):
204
- generator = llm.generate(prompt, sp, request_id=f"test{i}")
250
+ for i, (prompt, sp,
251
+ lr) in enumerate(zip(prompts, sampling_params, lora_requests)):
252
+ generator = llm.generate(prompt,
253
+ sp,
254
+ lora_request=lr,
255
+ request_id=f"test{i}")
256
generators.append(generator)
257
all_gens = merge_async_iterators(*generators)
258
async for i, res in all_gens:
@@ -297,6 +348,14 @@ def main(args: argparse.Namespace):
348
vocab_size = tokenizer.vocab_size
349
requests = []
350
for _ in range(args.num_prompts):
351
+
352
+ request_tokenizer = tokenizer
353
+ lora_request: Optional[LoRARequest] = None
354
+ if args.enable_lora:
355
+ lora_request, lora_tokenizer = get_random_lora_request(args)
356
+ if lora_tokenizer:
357
+ request_tokenizer = lora_tokenizer
358
+
359
# Synthesize a prompt with the given input length.
360
candidate_ids = [
361
random.randint(0, vocab_size - 1)
@@ -305,8 +364,8 @@ def main(args: argparse.Namespace):
364
# As tokenizer may add additional tokens like BOS, we need to try
365
# different lengths to get the desired input length.
366
for _ in range(5): # Max attempts to correct
308
- candidate_prompt = tokenizer.decode(candidate_ids)
309
- tokenized_len = len(tokenizer.encode(candidate_prompt))
367
+ candidate_prompt = request_tokenizer.decode(candidate_ids)
368
+ tokenized_len = len(request_tokenizer.encode(candidate_prompt))
369
370
if tokenized_len == args.input_len:
371
break
@@ -323,7 +382,8 @@ def main(args: argparse.Namespace):
382
requests.append(
383
SampleRequest(prompt=candidate_prompt,
384
prompt_len=args.input_len,
326
- expected_output_len=args.output_len))
385
+ expected_output_len=args.output_len,
386
+ lora_request=lora_request))
387
else:
388
requests = sample_requests(tokenizer, args)
389
@@ -422,6 +482,14 @@ if __name__ == "__main__":
482
action='store_true',
483
default=False,
484
help="Disable decoupled async engine frontend.")
485
+ # LoRA
486
+ parser.add_argument(
487
+ "--lora-path",
488
+ type=str,
489
+ default=None,
490
+ help="Path to the lora adapters to use. This can be an absolute path, "
491
+ "a relative path, or a Hugging Face model identifier.")
492
+
493
parser = AsyncEngineArgs.add_cli_args(parser)
494
args = parser.parse_args()
495
if args.tokenizer is None:
@@ -431,6 +499,8 @@ if __name__ == "__main__":
499
assert args.output_len is not None
500
else:
501
assert args.input_len is None
502
+ if args.enable_lora:
503
+ assert args.lora_path is not None
504
505
if args.backend == "vllm":
506
if args.hf_max_batch_size is not None:
@@ -440,6 +510,9 @@ if __name__ == "__main__":
510
raise ValueError("HF max batch size is required for HF backend.")
511
if args.quantization is not None:
512
raise ValueError("Quantization is only for vLLM backend.")
513
+ if args.enable_lora is not None:
514
+ raise ValueError("LoRA benchmarking is only supported for vLLM"
515
+ " backend")
516
elif args.backend == "mii":
517
if args.dtype != "auto":
518
raise ValueError("dtype must be auto for MII backend.")
@@ -452,4 +525,7 @@ if __name__ == "__main__":
525
if args.tokenizer != args.model:
526
raise ValueError("Tokenizer must be the same as the model for MII "
527
"backend.")
528
+ if args.enable_lora is not None:
529
+ raise ValueError("LoRA benchmarking is only supported for vLLM"
530
+ " backend")
531
main(args)