A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Benchmark] Support sample from HF datasets and image input for benchmark_serving (#8495)
Isotr0py committed
Sep 17, 2024 at 15:34 UTC
1b6de8352b878348974b3f117cbb68ed18daa609
2 files changed
+177
-68
benchmarks/backend_request_func.py
+5
-1
@@ -25,6 +25,7 @@ class RequestFuncInput:
25
best_of: int = 1
26
use_beam_search: bool = False
27
logprobs: Optional[int] = None
28
+ multi_modal_content: Optional[dict] = None
29
30
31
@dataclass
@@ -312,12 +313,15 @@ async def async_request_openai_chat_completions(
313
314
async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
315
assert not request_func_input.use_beam_search
316
+ content = [{"type": "text", "text": request_func_input.prompt}]
317
+ if request_func_input.multi_modal_content:
318
+ content.append(request_func_input.multi_modal_content)
319
payload = {
320
"model": request_func_input.model,
321
"messages": [
322
{
323
"role": "user",
320
- "content": request_func_input.prompt,
324
+ "content": content
325
},
326
],
327
"temperature": 0.0,
benchmarks/benchmark_serving.py
+172
-67
@@ -24,6 +24,8 @@ On the client side, run:
24
"""
25
import argparse
26
import asyncio
27
+import base64
28
+import io
29
import json
30
import os
31
import random
@@ -31,11 +33,13 @@ import time
33
import warnings
34
from dataclasses import dataclass
35
from datetime import datetime
34
-from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple
36
+from typing import Any, AsyncGenerator, Collection, Dict, List, Optional, Tuple
37
38
import numpy as np
39
from backend_request_func import (ASYNC_REQUEST_FUNCS, RequestFuncInput,
40
RequestFuncOutput)
41
+from datasets import load_dataset
42
+from PIL.Image import Image
43
from tqdm.asyncio import tqdm
44
from transformers import PreTrainedTokenizerBase
45
@@ -84,7 +88,7 @@ def sample_sharegpt_requests(
88
num_requests: int,
89
tokenizer: PreTrainedTokenizerBase,
90
fixed_output_len: Optional[int] = None,
87
-) -> List[Tuple[str, int, int]]:
91
+) -> List[Tuple[str, int, int, None]]:
92
if fixed_output_len is not None and fixed_output_len < 4:
93
raise ValueError("output_len too small")
94
# Load the dataset.
@@ -119,7 +123,7 @@ def sample_sharegpt_requests(
123
if prompt_len > 1024 or prompt_len + output_len > 2048:
124
# Prune too long sequences.
125
continue
122
- filtered_dataset.append((prompt, prompt_len, output_len))
126
+ filtered_dataset.append((prompt, prompt_len, output_len, None))
127
128
return filtered_dataset
129
@@ -131,7 +135,7 @@ def sample_sonnet_requests(
135
output_len: int,
136
prefix_len: int,
137
tokenizer: PreTrainedTokenizerBase,
134
-) -> List[Tuple[str, str, int, int]]:
138
+) -> List[Tuple[str, str, int, int, None]]:
139
assert (
140
input_len > prefix_len
141
), "'args.sonnet-input-len' must be greater than 'args.prefix-input-len'."
@@ -189,7 +193,65 @@ def sample_sonnet_requests(
193
message, add_generation_prompt=True, tokenize=False)
194
prompt_len = len(tokenizer(prompt_formatted).input_ids)
195
sampled_requests.append(
192
- (prompt, prompt_formatted, prompt_len, output_len))
196
+ (prompt, prompt_formatted, prompt_len, output_len, None))
197
+
198
+ return sampled_requests
199
+
200
+
201
+def sample_hf_requests(
202
+ dataset_path: str,
203
+ dataset_subset: str,
204
+ dataset_split: str,
205
+ num_requests: int,
206
+ tokenizer: PreTrainedTokenizerBase,
207
+ fixed_output_len: Optional[int] = None,
208
+) -> List[Tuple[str, str, int, Optional[Dict[str, Collection[str]]]]]:
209
+ dataset = load_dataset(dataset_path,
210
+ name=dataset_subset,
211
+ split=dataset_split,
212
+ streaming=True)
213
+ assert "conversations" in dataset.features, (
214
+ "HF Dataset must have 'conversations' column.")
215
+ filtered_dataset = dataset.shuffle().filter(
216
+ lambda x: len(x["conversations"]) >= 2)
217
+ sampled_requests: List[Tuple[str, int, int, Dict[str,
218
+ Collection[str]]]] = []
219
+ for data in filtered_dataset:
220
+ if len(sampled_requests) == num_requests:
221
+ break
222
+
223
+ # Tokenize the prompts and completions.
224
+ prompt = data["conversations"][0]["value"]
225
+ prompt_token_ids = tokenizer(prompt).input_ids
226
+ completion = data["conversations"][1]["value"]
227
+ completion_token_ids = tokenizer(completion).input_ids
228
+ prompt_len = len(prompt_token_ids)
229
+ output_len = len(completion_token_ids
230
+ ) if fixed_output_len is None else fixed_output_len
231
+ if prompt_len < 4 or output_len < 4:
232
+ # Prune too short sequences.
233
+ continue
234
+ if prompt_len > 1024 or prompt_len + output_len > 2048:
235
+ # Prune too long sequences.
236
+ continue
237
+
238
+ if "image" in data and isinstance(data["image"], Image):
239
+ image: Image = data["image"]
240
+ image = image.convert("RGB")
241
+ image_data = io.BytesIO()
242
+ image.save(image_data, format='JPEG')
243
+ image_base64 = base64.b64encode(
244
+ image_data.getvalue()).decode("utf-8")
245
+ mm_content = {
246
+ "type": "image_url",
247
+ "image_url": {
248
+ "url": f"data:image/jpeg;base64,{image_base64}"
249
+ },
250
+ }
251
+ else:
252
+ mm_content = None
253
+
254
+ sampled_requests.append((prompt, prompt_len, output_len, mm_content))
255
256
return sampled_requests
257
@@ -223,8 +285,8 @@ def sample_random_requests(
285
[(offsets[i] + i + j) % tokenizer.vocab_size
286
for j in range(input_lens[i])])
287
226
- input_requests.append(
227
- (prompt, int(prefix_len + input_lens[i]), int(output_lens[i])))
288
+ input_requests.append((prompt, int(prefix_len + input_lens[i]),
289
+ int(output_lens[i]), None))
290
291
return input_requests
292
@@ -343,7 +405,12 @@ async def benchmark(
405
raise ValueError(f"Unknown backend: {backend}")
406
407
print("Starting initial single prompt test run...")
346
- test_prompt, test_prompt_len, test_output_len = input_requests[0]
408
+ test_prompt, test_prompt_len, test_output_len, test_mm_content = (
409
+ input_requests[0])
410
+ if backend != "openai-chat" and test_mm_content is not None:
411
+ # multi-modal benchmark is only available on OpenAI Chat backend.
412
+ raise ValueError(
413
+ "Multi-modal content is only supported on 'openai-chat' backend.")
414
test_input = RequestFuncInput(
415
model=model_id,
416
prompt=test_prompt,
@@ -353,6 +420,7 @@ async def benchmark(
420
logprobs=logprobs,
421
best_of=best_of,
422
use_beam_search=use_beam_search,
423
+ multi_modal_content=test_mm_content,
424
)
425
test_output = await request_func(request_func_input=test_input)
426
if not test_output.success:
@@ -373,6 +441,7 @@ async def benchmark(
441
logprobs=logprobs,
442
best_of=best_of,
443
use_beam_search=use_beam_search,
444
+ multi_modal_content=test_mm_content,
445
)
446
profile_output = await request_func(request_func_input=profile_input)
447
if profile_output.success:
@@ -385,7 +454,7 @@ async def benchmark(
454
benchmark_start_time = time.perf_counter()
455
tasks: List[asyncio.Task] = []
456
async for request in get_request(input_requests, request_rate):
388
- prompt, prompt_len, output_len = request
457
+ prompt, prompt_len, output_len, mm_content = request
458
request_func_input = RequestFuncInput(
459
model=model_id,
460
prompt=prompt,
@@ -395,6 +464,7 @@ async def benchmark(
464
logprobs=logprobs,
465
best_of=best_of,
466
use_beam_search=use_beam_search,
467
+ multi_modal_content=mm_content,
468
)
469
tasks.append(
470
asyncio.create_task(
@@ -575,6 +645,16 @@ def main(args: argparse.Namespace):
645
for prompt, prompt_formatted, prompt_len,
646
output_len in input_requests]
647
648
+ elif args.dataset_name == "hf":
649
+ input_requests = sample_hf_requests(
650
+ dataset_path=args.dataset_path,
651
+ dataset_subset=args.hf_subset,
652
+ dataset_split=args.hf_split,
653
+ num_requests=args.num_prompts,
654
+ tokenizer=tokenizer,
655
+ fixed_output_len=args.hf_output_len,
656
+ )
657
+
658
elif args.dataset_name == "random":
659
input_requests = sample_random_requests(
660
prefix_len=args.random_prefix_len,
@@ -685,13 +765,14 @@ if __name__ == "__main__":
765
"--dataset-name",
766
type=str,
767
default="sharegpt",
688
- choices=["sharegpt", "sonnet", "random"],
768
+ choices=["sharegpt", "sonnet", "random", "hf"],
769
help="Name of the dataset to benchmark on.",
770
)
771
parser.add_argument("--dataset-path",
772
type=str,
773
default=None,
694
- help="Path to the dataset.")
774
+ help="Path to the sharegpt/sonnet dataset. "
775
+ "Or the huggingface dataset ID if using HF dataset.")
776
parser.add_argument(
777
"--model",
778
type=str,
@@ -718,26 +799,6 @@ if __name__ == "__main__":
799
default=1000,
800
help="Number of prompts to process.",
801
)
721
- parser.add_argument(
722
- "--sharegpt-output-len",
723
- type=int,
724
- default=None,
725
- help="Output length for each request. Overrides the output length "
726
- "from the ShareGPT dataset.")
727
- parser.add_argument(
728
- "--sonnet-input-len",
729
- type=int,
730
- default=550,
731
- help=
732
- "Number of input tokens per request, used only for sonnet dataset.",
733
- )
734
- parser.add_argument(
735
- "--sonnet-output-len",
736
- type=int,
737
- default=150,
738
- help=
739
- "Number of output tokens per request, used only for sonnet dataset.",
740
- )
802
parser.add_argument(
803
"--logprobs",
804
type=int,
@@ -748,42 +809,6 @@ if __name__ == "__main__":
809
"logprob is returned for each token; or (2) if beam search "
810
"is enabled 1 logprob per token is computed"),
811
)
751
- parser.add_argument(
752
- "--sonnet-prefix-len",
753
- type=int,
754
- default=200,
755
- help=
756
- "Number of prefix tokens per request, used only for sonnet dataset.",
757
- )
758
- parser.add_argument(
759
- "--random-input-len",
760
- type=int,
761
- default=1024,
762
- help=
763
- "Number of input tokens per request, used only for random sampling.",
764
- )
765
- parser.add_argument(
766
- "--random-output-len",
767
- type=int,
768
- default=128,
769
- help=
770
- "Number of output tokens per request, used only for random sampling.",
771
- )
772
- parser.add_argument(
773
- "--random-range-ratio",
774
- type=float,
775
- default=1.0,
776
- help="Range of sampled ratio of input/output length, "
777
- "used only for random sampling.",
778
- )
779
- parser.add_argument(
780
- "--random-prefix-len",
781
- type=int,
782
- default=0,
783
- help="Number of fixed prefix tokens before random "
784
- " context. The length range of context in a random "
785
- " request is [random-prefix-len, "
786
- " random-prefix-len + random-prefix-len * random-range-ratio).")
812
parser.add_argument(
813
"--request-rate",
814
type=float,
@@ -857,5 +882,85 @@ if __name__ == "__main__":
882
"Use \"--percentile-metrics\" to select metrics.",
883
)
884
885
+ # group for dataset specific arguments
886
+ sonnet_group = parser.add_argument_group("sonnet dataset options")
887
+ sonnet_group.add_argument(
888
+ "--sonnet-input-len",
889
+ type=int,
890
+ default=550,
891
+ help=
892
+ "Number of input tokens per request, used only for sonnet dataset.",
893
+ )
894
+ sonnet_group.add_argument(
895
+ "--sonnet-output-len",
896
+ type=int,
897
+ default=150,
898
+ help=
899
+ "Number of output tokens per request, used only for sonnet dataset.",
900
+ )
901
+ sonnet_group.add_argument(
902
+ "--sonnet-prefix-len",
903
+ type=int,
904
+ default=200,
905
+ help=
906
+ "Number of prefix tokens per request, used only for sonnet dataset.",
907
+ )
908
+
909
+ sharegpt_group = parser.add_argument_group("sharegpt dataset options")
910
+ sharegpt_group.add_argument(
911
+ "--sharegpt-output-len",
912
+ type=int,
913
+ default=None,
914
+ help="Output length for each request. Overrides the output length "
915
+ "from the ShareGPT dataset.")
916
+
917
+ random_group = parser.add_argument_group("random dataset options")
918
+ random_group.add_argument(
919
+ "--random-input-len",
920
+ type=int,
921
+ default=1024,
922
+ help=
923
+ "Number of input tokens per request, used only for random sampling.",
924
+ )
925
+ random_group.add_argument(
926
+ "--random-output-len",
927
+ type=int,
928
+ default=128,
929
+ help=
930
+ "Number of output tokens per request, used only for random sampling.",
931
+ )
932
+ random_group.add_argument(
933
+ "--random-range-ratio",
934
+ type=float,
935
+ default=1.0,
936
+ help="Range of sampled ratio of input/output length, "
937
+ "used only for random sampling.",
938
+ )
939
+ random_group.add_argument(
940
+ "--random-prefix-len",
941
+ type=int,
942
+ default=0,
943
+ help="Number of fixed prefix tokens before random "
944
+ " context. The length range of context in a random "
945
+ " request is [random-prefix-len, "
946
+ " random-prefix-len + random-prefix-len * random-range-ratio).")
947
+
948
+ hf_group = parser.add_argument_group("hf dataset options")
949
+ hf_group.add_argument("--hf-subset",
950
+ type=str,
951
+ default=None,
952
+ help="Subset of the HF dataset.")
953
+ hf_group.add_argument("--hf-split",
954
+ type=str,
955
+ default=None,
956
+ help="Split of the HF dataset.")
957
+ hf_group.add_argument(
958
+ "--hf-output-len",
959
+ type=int,
960
+ default=None,
961
+ help="Output length for each request. Overrides the output lengths "
962
+ "from the sampled HF dataset.",
963
+ )
964
+
965
args = parser.parse_args()
966
main(args)