@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0c6f998554
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Benchmark] Add benchmark script for CPU offloading (#11533)

Signed-off-by: ApostaC <yihua98@uchicago.edu> Co-authored-by: KuntaiDu <kuntai@uchicago.edu>

Yihua Cheng committed Dec 31, 2024 at 18:10 UTC 0c6f9985547d6b510d34c6c873db54abe03eb346
1 file changed +184
benchmarks/benchmark_long_document_qa_throughput.py new
+184
@@ -0,0 +1,184 @@
1 +"""
2 +Offline benchmark to test the long document QA throughput.
3 +
4 +Example usage:
5 + # This command run the vllm with 50GB CPU memory for offloading
6 + # The workload samples 8 different prompts with a default input
7 + # length of 20000 tokens, then replicates each prompt 2 times
8 + # in random order.
9 + python benchmark_long_document_qa_throughput.py \
10 + --model meta-llama/Llama-2-7b-chat-hf \
11 + --enable-prefix-caching \
12 + --num-documents 8 \
13 + --repeat-count 2
14 +
15 +Commandline arguments:
16 + --num-documents: The number of documents to sample prompts from.
17 +
18 + --document-length: The length of each document in tokens.
19 + (Optional, default: 20000)
20 +
21 + --output-len: The number of tokens to generate for each prompt.
22 + (Optional, default: 10)
23 +
24 + --repeat-count: The number of times to repeat each prompt.
25 + (Optional, default: 2)
26 +
27 + --repeat-mode: The mode to repeat prompts. The supported modes are:
28 + - 'random': shuffle the prompts randomly. (Default)
29 + - 'tile': the entire prompt list is repeated in sequence. (Potentially
30 + lowest cache hit)
31 + - 'interleave': each prompt is repeated consecutively before
32 + moving to the next element. (Highest cache hit)
33 +
34 + --shuffle-seed: Random seed when the repeat mode is "random".
35 + (Optional, default: 0)
36 +
37 +In the meantime, it also supports all the vLLM engine args to initialize the
38 +LLM engine. You can refer to the `vllm.engine.arg_utils.EngineArgs` for more
39 +details.
40 +"""
41 +
42 +import dataclasses
43 +import random
44 +import time
45 +
46 +from vllm import LLM, SamplingParams
47 +from vllm.engine.arg_utils import EngineArgs
48 +from vllm.utils import FlexibleArgumentParser
49 +
50 +
51 +def test_long_document_qa(llm=None, sampling_params=None, prompts=None):
52 + """
53 + Test long document QA with the given prompts and sampling parameters.
54 + Print the time spent in processing all the prompts.
55 +
56 + Args:
57 + llm: The language model used for generating responses.
58 + sampling_params: Sampling parameter used to generate the response.
59 + prompts: A list of prompt strings to be processed by the LLM.
60 + """
61 + start_time = time.time()
62 + llm.generate(prompts, sampling_params=sampling_params)
63 + end_time = time.time()
64 + print(f"Time to execute all requests: {end_time - start_time:.4f} secs")
65 +
66 +
67 +def repeat_prompts(prompts, repeat_count, mode: str):
68 + """
69 + Repeat each prompt in the list for a specified number of times.
70 + The order of prompts in the output list depends on the mode.
71 +
72 + Args:
73 + prompts: A list of prompts to be repeated.
74 + repeat_count: The number of times each prompt is repeated.
75 + mode: The mode of repetition. Supported modes are:
76 + - 'random': Shuffle the prompts randomly after repetition.
77 + - 'tile': Repeat the entire prompt list in sequence.
78 + Example: [1, 2, 3] -> [1, 2, 3, 1, 2, 3].
79 + - 'interleave': Repeat each prompt consecutively before moving to
80 + the next. Example: [1, 2, 3] -> [1, 1, 2, 2, 3, 3].
81 +
82 + Returns:
83 + A list of repeated prompts in the specified order.
84 +
85 + Raises:
86 + ValueError: If an invalid mode is provided.
87 + """
88 + print("Repeat mode: ", mode)
89 + if mode == 'random':
90 + repeated_prompts = prompts * repeat_count
91 + random.shuffle(repeated_prompts)
92 + return repeated_prompts
93 + elif mode == 'tile':
94 + return prompts * repeat_count
95 + elif mode == 'interleave':
96 + repeated_prompts = []
97 + for prompt in prompts:
98 + repeated_prompts.extend([prompt] * repeat_count)
99 + return repeated_prompts
100 + else:
101 + raise ValueError(f"Invalid mode: {mode}, only support "
102 + "'random', 'tile', 'interleave'")
103 +
104 +
105 +def main(args):
106 + random.seed(args.shuffle_seed)
107 +
108 + # Prepare the prompts:
109 + # we append the document id at the beginning to avoid any of the document
110 + # being the prefix of other documents
111 + prompts = [
112 + str(i) + ' '.join(['hi'] * args.document_length)
113 + for i in range(args.num_documents)
114 + ]
115 +
116 + prompts = repeat_prompts(prompts, args.repeat_count, mode=args.repeat_mode)
117 +
118 + warmup_prompts = [
119 + "This is warm up request " + str(i) + \
120 + ' '.join(['hi'] * args.document_length)
121 + for i in range(args.num_documents)]
122 +
123 + # Create the LLM engine
124 + engine_args = EngineArgs.from_cli_args(args)
125 + llm = LLM(**dataclasses.asdict(engine_args))
126 + sampling_params = SamplingParams(temperature=0, max_tokens=args.output_len)
127 +
128 + print("------warm up------")
129 + test_long_document_qa(
130 + llm=llm,
131 + prompts=warmup_prompts,
132 + sampling_params=sampling_params,
133 + )
134 +
135 + print("------start generating------")
136 + test_long_document_qa(
137 + llm=llm,
138 + prompts=prompts,
139 + sampling_params=sampling_params,
140 + )
141 +
142 +
143 +if __name__ == "__main__":
144 + parser = FlexibleArgumentParser(
145 + description=
146 + 'Benchmark the performance with or without automatic prefix caching.')
147 +
148 + parser.add_argument(
149 + '--document-length',
150 + type=int,
151 + # Roughly the number of tokens for a system paper,
152 + # excluding images
153 + default=20000,
154 + help='Range of input lengths for sampling prompts,'
155 + 'specified as "min:max" (e.g., "128:256").')
156 +
157 + parser.add_argument('--num-documents',
158 + type=int,
159 + default=8,
160 + help='Range of input lengths for sampling prompts,'
161 + 'specified as "min:max" (e.g., "128:256").')
162 +
163 + parser.add_argument('--output-len', type=int, default=10)
164 +
165 + parser.add_argument('--repeat-count',
166 + type=int,
167 + default=2,
168 + help='Number of times to repeat each prompt')
169 +
170 + parser.add_argument("--repeat-mode",
171 + type=str,
172 + default='random',
173 + help='The mode to repeat prompts. The supported '
174 + 'modes are "random", "tile", and "interleave". '
175 + 'See repeat_prompts() in the source code for details.')
176 +
177 + parser.add_argument("--shuffle-seed",
178 + type=int,
179 + default=0,
180 + help='Random seed when the repeat mode is "random"')
181 +
182 + parser = EngineArgs.add_cli_args(parser)
183 + args = parser.parse_args()
184 + main(args)