A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[misc] partial prefix & random input generation benchmark (#9929)
Signed-off-by: rickyx <rickyx@anyscale.com>
Ricky Xu committed
Nov 18, 2024 at 15:39 UTC
90a6c759caf84ff7722449a33895e397ccf1a2af
1 file changed
+91
-25
benchmarks/benchmark_prefix_caching.py
+91
-25
@@ -54,13 +54,30 @@ def test_prefix(llm=None, sampling_params=None, prompts=None):
54
print(f"cost time {end_time - start_time}")
55
56
57
-def sample_requests(
57
+@dataclasses.dataclass
58
+class Request:
59
+ prompt: str
60
+ prompt_len: int
61
+ output_len: int
62
+
63
+
64
+def sample_tokens(tokenizer: PreTrainedTokenizerBase, length: int) -> str:
65
+ vocab = tokenizer.get_vocab()
66
+ # Remove the special tokens.
67
+ vocab = {
68
+ k: v
69
+ for k, v in vocab.items() if k not in tokenizer.all_special_ids
70
+ }
71
+ return random.choices(list(vocab.values()), k=length)
72
+
73
+
74
+def sample_requests_from_dataset(
75
dataset_path: str,
76
num_requests: int,
77
tokenizer: PreTrainedTokenizerBase,
78
input_length_range: Tuple[int, int],
79
fixed_output_len: Optional[int],
63
-) -> List[Tuple[str, int, int]]:
80
+) -> List[Request]:
81
if fixed_output_len is not None and fixed_output_len < 4:
82
raise ValueError("output_len too small")
83
@@ -77,31 +94,55 @@ def sample_requests(
94
random.shuffle(dataset)
95
96
min_len, max_len = input_length_range
97
+ assert min_len >= 0 and max_len >= min_len, "input_length_range too small"
98
99
# Filter out sequences that are too long or too short
82
- filtered_dataset: List[Tuple[str, int, int]] = []
100
+ filtered_requests: List[Request] = []
101
+
102
for i in range(len(dataset)):
84
- if len(filtered_dataset) == num_requests:
103
+ if len(filtered_requests) == num_requests:
104
break
105
106
# Tokenize the prompts and completions.
88
- prompt = dataset[i][0]
89
- prompt_token_ids = tokenizer(prompt).input_ids
107
+ prompt_token_ids = tokenizer(dataset[i][0]).input_ids
108
+ prompt = tokenizer.decode(prompt_token_ids)
109
completion = dataset[i][1]
110
completion_token_ids = tokenizer(completion).input_ids
111
prompt_len = len(prompt_token_ids)
93
- output_len = len(completion_token_ids
94
- ) if fixed_output_len is None else fixed_output_len
95
- if prompt_len < 4 or output_len < 4:
96
- # Prune too short sequences.
97
- continue
112
+ output_len = (len(completion_token_ids)
113
+ if fixed_output_len is None else fixed_output_len)
114
if min_len <= prompt_len <= max_len:
99
- filtered_dataset.append((prompt, prompt_len, output_len))
115
+ filtered_requests.append(Request(prompt, prompt_len, output_len))
116
+
117
+ return filtered_requests
118
+
119
+
120
+def sample_requests_from_random(
121
+ num_requests: int,
122
+ tokenizer: PreTrainedTokenizerBase,
123
+ input_length_range: Tuple[int, int],
124
+ fixed_output_len: Optional[int],
125
+ prefix_len: int,
126
+) -> List[Request]:
127
101
- return filtered_dataset
128
+ requests = []
129
+ prefix_token_ids = sample_tokens(tokenizer, prefix_len)
130
+ min_len, max_len = input_length_range
131
+
132
+ for i in range(num_requests):
133
+ unique_part_token_ids = sample_tokens(
134
+ tokenizer,
135
+ random.randint(min_len - prefix_len, max_len - prefix_len))
136
+ prompt_token_ids = prefix_token_ids + unique_part_token_ids
137
+ prompt = tokenizer.decode(prompt_token_ids)
138
+ prompt_len = len(prompt_token_ids)
139
+ assert (min_len <= prompt_len <= max_len
140
+ ), f"prompt_len {prompt_len} out of range {min_len}:{max_len}"
141
+ requests.append(Request(prompt, prompt_len, fixed_output_len))
142
+ return requests
143
144
104
-def repeat_and_sort_requests(requests: List[Tuple[str, int, int]],
145
+def repeat_and_sort_requests(requests: List[Request],
146
repeat_count: int,
147
sort: bool = False) -> List[str]:
148
repeated_requests = requests * repeat_count
@@ -109,7 +150,7 @@ def repeat_and_sort_requests(requests: List[Tuple[str, int, int]],
150
repeated_requests.sort(key=lambda x: x[1])
151
else:
152
random.shuffle(repeated_requests)
112
- return [req[0] for req in repeated_requests]
153
+ return [req.prompt for req in repeated_requests]
154
155
156
def main(args):
@@ -117,9 +158,12 @@ def main(args):
158
input_length_range = tuple(map(int, args.input_length_range.split(':')))
159
random.seed(args.seed)
160
if args.dataset_path is not None:
120
- print(f"Start to sample {args.num_prompts} prompts"
161
+ if args.prefix_len > 0:
162
+ raise ValueError("prefix-len is not supported when "
163
+ "dataset-path is provided.")
164
+ print(f"Start to sample {args.num_prompts} prompts "
165
f"from {args.dataset_path}")
122
- filtered_datasets = sample_requests(
166
+ filtered_requests = sample_requests_from_dataset(
167
dataset_path=args.dataset_path,
168
num_requests=args.num_prompts,
169
tokenizer=tokenizer,
@@ -127,9 +171,22 @@ def main(args):
171
fixed_output_len=args.output_len,
172
)
173
else:
130
- prompt_len = len(tokenizer(PROMPT).input_ids)
131
- filtered_datasets = [(PROMPT, prompt_len, args.output_len)
132
- ] * args.num_prompts
174
+ print(f"Start to sample {args.num_prompts} prompts from random")
175
+ filtered_requests = sample_requests_from_random(
176
+ num_requests=args.num_prompts,
177
+ tokenizer=tokenizer,
178
+ input_length_range=input_length_range,
179
+ fixed_output_len=args.output_len,
180
+ prefix_len=args.prefix_len,
181
+ )
182
+
183
+ # Print some helpful stats of the requests.
184
+ print(f"Sampled {len(filtered_requests)} requests.")
185
+ prompt_lens = [req.prompt_len for req in filtered_requests]
186
+ print(f"Average input length: {sum(prompt_lens) / len(prompt_lens)}")
187
+ print(f"P50 input length: {sorted(prompt_lens)[len(prompt_lens) // 2]}")
188
+ print(f"Min Prompt Length: {min(prompt_lens)}")
189
+ print(f"Max Prompt Length: {max(prompt_lens)}")
190
191
engine_args = EngineArgs.from_cli_args(args)
192
@@ -137,8 +194,8 @@ def main(args):
194
195
sampling_params = SamplingParams(temperature=0, max_tokens=args.output_len)
196
140
- print("Testing filtered datasets")
141
- prompts = repeat_and_sort_requests(filtered_datasets,
197
+ print("Testing filtered requests")
198
+ prompts = repeat_and_sort_requests(filtered_requests,
199
repeat_count=args.repeat_count,
200
sort=args.sort)
201
@@ -161,20 +218,29 @@ if __name__ == "__main__":
218
parser.add_argument('--output-len', type=int, default=10)
219
parser.add_argument('--num-prompts',
220
type=int,
164
- default=1,
221
+ required=True,
222
help="Number of the prompts sampled from dataset")
223
parser.add_argument('--repeat-count',
224
type=int,
168
- default=100,
225
+ default=1,
226
help='Number of times to repeat each prompt')
227
parser.add_argument('--sort',
228
action='store_true',
229
help='Sort prompts by input length')
230
parser.add_argument('--input-length-range',
231
type=str,
175
- default='128:256',
232
+ required=True,
233
help='Range of input lengths for sampling prompts,'
234
'specified as "min:max" (e.g., "128:256").')
235
+ parser.add_argument(
236
+ "--prefix-len",
237
+ type=int,
238
+ default=0,
239
+ help="Specifies the length of a common prefix to be "
240
+ "added to the input prompt. The input-length-range will "
241
+ "subtract this length when filtering prompts. Only used "
242
+ "when dataset-path is not provided.",
243
+ )
244
245
parser = EngineArgs.add_cli_args(parser)
246
args = parser.parse_args()