A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
Enable Random Prefix Caching in Serving Profiling Tool (benchmark_serving.py) (#8241)
Wei-Sheng Chin committed
Sep 6, 2024 at 20:18 UTC
795b662cffe79fa0fa9a3f13a65113abdb4f96a9
1 file changed
+23
-4
benchmarks/benchmark_serving.py
+23
-4
@@ -195,8 +195,16 @@ def sample_sonnet_requests(
195
196
197
def sample_random_requests(
198
- input_len: int, output_len: int, num_prompts: int, range_ratio: float,
199
- tokenizer: PreTrainedTokenizerBase) -> List[Tuple[str, int, int]]:
198
+ prefix_len: int,
199
+ input_len: int,
200
+ output_len: int,
201
+ num_prompts: int,
202
+ range_ratio: float,
203
+ tokenizer: PreTrainedTokenizerBase,
204
+) -> List[Tuple[str, int, int]]:
205
+ prefix_token_ids = np.random.randint(0,
206
+ tokenizer.vocab_size,
207
+ size=prefix_len).tolist()
208
209
input_lens = np.random.randint(
210
int(input_len * range_ratio),
@@ -211,10 +219,12 @@ def sample_random_requests(
219
offsets = np.random.randint(0, tokenizer.vocab_size, size=num_prompts)
220
input_requests = []
221
for i in range(num_prompts):
214
- prompt = tokenizer.decode([(offsets[i] + i + j) % tokenizer.vocab_size
222
+ prompt = tokenizer.decode(prefix_token_ids +
223
+ [(offsets[i] + i + j) % tokenizer.vocab_size
224
for j in range(input_lens[i])])
225
+
226
input_requests.append(
217
- (prompt, int(input_lens[i]), int(output_lens[i])))
227
+ (prompt, int(prefix_len + input_lens[i]), int(output_lens[i])))
228
229
return input_requests
230
@@ -567,6 +577,7 @@ def main(args: argparse.Namespace):
577
578
elif args.dataset_name == "random":
579
input_requests = sample_random_requests(
580
+ prefix_len=args.random_prefix_len,
581
input_len=args.random_input_len,
582
output_len=args.random_output_len,
583
num_prompts=args.num_prompts,
@@ -765,6 +776,14 @@ if __name__ == "__main__":
776
help="Range of sampled ratio of input/output length, "
777
"used only for random sampling.",
778
)
779
+ parser.add_argument(
780
+ "--random-prefix-len",
781
+ type=int,
782
+ default=0,
783
+ help="Number of fixed prefix tokens before random "
784
+ " context. The length range of context in a random "
785
+ " request is [random-prefix-len, "
786
+ " random-prefix-len + random-prefix-len * random-range-ratio).")
787
parser.add_argument(
788
"--request-rate",
789
type=float,