@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 90a6c759ca
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[misc] partial prefix & random input generation benchmark (#9929)

Signed-off-by: rickyx <rickyx@anyscale.com>

Ricky Xu committed Nov 18, 2024 at 15:39 UTC 90a6c759caf84ff7722449a33895e397ccf1a2af
1 file changed +91 -25
benchmarks/benchmark_prefix_caching.py
+91 -25
@@ -54,13 +54,30 @@ def test_prefix(llm=None, sampling_params=None, prompts=None):
54 print(f"cost time {end_time - start_time}")
55
56
57 -def sample_requests(
57 +@dataclasses.dataclass
58 +class Request:
59 + prompt: str
60 + prompt_len: int
61 + output_len: int
62 +
63 +
64 +def sample_tokens(tokenizer: PreTrainedTokenizerBase, length: int) -> str:
65 + vocab = tokenizer.get_vocab()
66 + # Remove the special tokens.
67 + vocab = {
68 + k: v
69 + for k, v in vocab.items() if k not in tokenizer.all_special_ids
70 + }
71 + return random.choices(list(vocab.values()), k=length)
72 +
73 +
74 +def sample_requests_from_dataset(
75 dataset_path: str,
76 num_requests: int,
77 tokenizer: PreTrainedTokenizerBase,
78 input_length_range: Tuple[int, int],
79 fixed_output_len: Optional[int],
63 -) -> List[Tuple[str, int, int]]:
80 +) -> List[Request]:
81 if fixed_output_len is not None and fixed_output_len < 4:
82 raise ValueError("output_len too small")
83
@@ -77,31 +94,55 @@ def sample_requests(
94 random.shuffle(dataset)
95
96 min_len, max_len = input_length_range
97 + assert min_len >= 0 and max_len >= min_len, "input_length_range too small"
98
99 # Filter out sequences that are too long or too short
82 - filtered_dataset: List[Tuple[str, int, int]] = []
100 + filtered_requests: List[Request] = []
101 +
102 for i in range(len(dataset)):
84 - if len(filtered_dataset) == num_requests:
103 + if len(filtered_requests) == num_requests:
104 break
105
106 # Tokenize the prompts and completions.
88 - prompt = dataset[i][0]
89 - prompt_token_ids = tokenizer(prompt).input_ids
107 + prompt_token_ids = tokenizer(dataset[i][0]).input_ids
108 + prompt = tokenizer.decode(prompt_token_ids)
109 completion = dataset[i][1]
110 completion_token_ids = tokenizer(completion).input_ids
111 prompt_len = len(prompt_token_ids)
93 - output_len = len(completion_token_ids
94 - ) if fixed_output_len is None else fixed_output_len
95 - if prompt_len < 4 or output_len < 4:
96 - # Prune too short sequences.
97 - continue
112 + output_len = (len(completion_token_ids)
113 + if fixed_output_len is None else fixed_output_len)
114 if min_len <= prompt_len <= max_len:
99 - filtered_dataset.append((prompt, prompt_len, output_len))
115 + filtered_requests.append(Request(prompt, prompt_len, output_len))
116 +
117 + return filtered_requests
118 +
119 +
120 +def sample_requests_from_random(
121 + num_requests: int,
122 + tokenizer: PreTrainedTokenizerBase,
123 + input_length_range: Tuple[int, int],
124 + fixed_output_len: Optional[int],
125 + prefix_len: int,
126 +) -> List[Request]:
127
101 - return filtered_dataset
128 + requests = []
129 + prefix_token_ids = sample_tokens(tokenizer, prefix_len)
130 + min_len, max_len = input_length_range
131 +
132 + for i in range(num_requests):
133 + unique_part_token_ids = sample_tokens(
134 + tokenizer,
135 + random.randint(min_len - prefix_len, max_len - prefix_len))
136 + prompt_token_ids = prefix_token_ids + unique_part_token_ids
137 + prompt = tokenizer.decode(prompt_token_ids)
138 + prompt_len = len(prompt_token_ids)
139 + assert (min_len <= prompt_len <= max_len
140 + ), f"prompt_len {prompt_len} out of range {min_len}:{max_len}"
141 + requests.append(Request(prompt, prompt_len, fixed_output_len))
142 + return requests
143
144
104 -def repeat_and_sort_requests(requests: List[Tuple[str, int, int]],
145 +def repeat_and_sort_requests(requests: List[Request],
146 repeat_count: int,
147 sort: bool = False) -> List[str]:
148 repeated_requests = requests * repeat_count
@@ -109,7 +150,7 @@ def repeat_and_sort_requests(requests: List[Tuple[str, int, int]],
150 repeated_requests.sort(key=lambda x: x[1])
151 else:
152 random.shuffle(repeated_requests)
112 - return [req[0] for req in repeated_requests]
153 + return [req.prompt for req in repeated_requests]
154
155
156 def main(args):
@@ -117,9 +158,12 @@ def main(args):
158 input_length_range = tuple(map(int, args.input_length_range.split(':')))
159 random.seed(args.seed)
160 if args.dataset_path is not None:
120 - print(f"Start to sample {args.num_prompts} prompts"
161 + if args.prefix_len > 0:
162 + raise ValueError("prefix-len is not supported when "
163 + "dataset-path is provided.")
164 + print(f"Start to sample {args.num_prompts} prompts "
165 f"from {args.dataset_path}")
122 - filtered_datasets = sample_requests(
166 + filtered_requests = sample_requests_from_dataset(
167 dataset_path=args.dataset_path,
168 num_requests=args.num_prompts,
169 tokenizer=tokenizer,
@@ -127,9 +171,22 @@ def main(args):
171 fixed_output_len=args.output_len,
172 )
173 else:
130 - prompt_len = len(tokenizer(PROMPT).input_ids)
131 - filtered_datasets = [(PROMPT, prompt_len, args.output_len)
132 - ] * args.num_prompts
174 + print(f"Start to sample {args.num_prompts} prompts from random")
175 + filtered_requests = sample_requests_from_random(
176 + num_requests=args.num_prompts,
177 + tokenizer=tokenizer,
178 + input_length_range=input_length_range,
179 + fixed_output_len=args.output_len,
180 + prefix_len=args.prefix_len,
181 + )
182 +
183 + # Print some helpful stats of the requests.
184 + print(f"Sampled {len(filtered_requests)} requests.")
185 + prompt_lens = [req.prompt_len for req in filtered_requests]
186 + print(f"Average input length: {sum(prompt_lens) / len(prompt_lens)}")
187 + print(f"P50 input length: {sorted(prompt_lens)[len(prompt_lens) // 2]}")
188 + print(f"Min Prompt Length: {min(prompt_lens)}")
189 + print(f"Max Prompt Length: {max(prompt_lens)}")
190
191 engine_args = EngineArgs.from_cli_args(args)
192
@@ -137,8 +194,8 @@ def main(args):
194
195 sampling_params = SamplingParams(temperature=0, max_tokens=args.output_len)
196
140 - print("Testing filtered datasets")
141 - prompts = repeat_and_sort_requests(filtered_datasets,
197 + print("Testing filtered requests")
198 + prompts = repeat_and_sort_requests(filtered_requests,
199 repeat_count=args.repeat_count,
200 sort=args.sort)
201
@@ -161,20 +218,29 @@ if __name__ == "__main__":
218 parser.add_argument('--output-len', type=int, default=10)
219 parser.add_argument('--num-prompts',
220 type=int,
164 - default=1,
221 + required=True,
222 help="Number of the prompts sampled from dataset")
223 parser.add_argument('--repeat-count',
224 type=int,
168 - default=100,
225 + default=1,
226 help='Number of times to repeat each prompt')
227 parser.add_argument('--sort',
228 action='store_true',
229 help='Sort prompts by input length')
230 parser.add_argument('--input-length-range',
231 type=str,
175 - default='128:256',
232 + required=True,
233 help='Range of input lengths for sampling prompts,'
234 'specified as "min:max" (e.g., "128:256").')
235 + parser.add_argument(
236 + "--prefix-len",
237 + type=int,
238 + default=0,
239 + help="Specifies the length of a common prefix to be "
240 + "added to the input prompt. The input-length-range will "
241 + "subtract this length when filtering prompts. Only used "
242 + "when dataset-path is not provided.",
243 + )
244
245 parser = EngineArgs.add_cli_args(parser)
246 args = parser.parse_args()