@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 795b662cff
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

Enable Random Prefix Caching in Serving Profiling Tool (benchmark_serving.py) (#8241)

Wei-Sheng Chin committed Sep 6, 2024 at 20:18 UTC 795b662cffe79fa0fa9a3f13a65113abdb4f96a9
1 file changed +23 -4
benchmarks/benchmark_serving.py
+23 -4
@@ -195,8 +195,16 @@ def sample_sonnet_requests(
195
196
197 def sample_random_requests(
198 - input_len: int, output_len: int, num_prompts: int, range_ratio: float,
199 - tokenizer: PreTrainedTokenizerBase) -> List[Tuple[str, int, int]]:
198 + prefix_len: int,
199 + input_len: int,
200 + output_len: int,
201 + num_prompts: int,
202 + range_ratio: float,
203 + tokenizer: PreTrainedTokenizerBase,
204 +) -> List[Tuple[str, int, int]]:
205 + prefix_token_ids = np.random.randint(0,
206 + tokenizer.vocab_size,
207 + size=prefix_len).tolist()
208
209 input_lens = np.random.randint(
210 int(input_len * range_ratio),
@@ -211,10 +219,12 @@ def sample_random_requests(
219 offsets = np.random.randint(0, tokenizer.vocab_size, size=num_prompts)
220 input_requests = []
221 for i in range(num_prompts):
214 - prompt = tokenizer.decode([(offsets[i] + i + j) % tokenizer.vocab_size
222 + prompt = tokenizer.decode(prefix_token_ids +
223 + [(offsets[i] + i + j) % tokenizer.vocab_size
224 for j in range(input_lens[i])])
225 +
226 input_requests.append(
217 - (prompt, int(input_lens[i]), int(output_lens[i])))
227 + (prompt, int(prefix_len + input_lens[i]), int(output_lens[i])))
228
229 return input_requests
230
@@ -567,6 +577,7 @@ def main(args: argparse.Namespace):
577
578 elif args.dataset_name == "random":
579 input_requests = sample_random_requests(
580 + prefix_len=args.random_prefix_len,
581 input_len=args.random_input_len,
582 output_len=args.random_output_len,
583 num_prompts=args.num_prompts,
@@ -765,6 +776,14 @@ if __name__ == "__main__":
776 help="Range of sampled ratio of input/output length, "
777 "used only for random sampling.",
778 )
779 + parser.add_argument(
780 + "--random-prefix-len",
781 + type=int,
782 + default=0,
783 + help="Number of fixed prefix tokens before random "
784 + " context. The length range of context in a random "
785 + " request is [random-prefix-len, "
786 + " random-prefix-len + random-prefix-len * random-range-ratio).")
787 parser.add_argument(
788 "--request-rate",
789 type=float,