@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0cc566ca8f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Misc] Add random seed for prefix cache benchmark (#9081)

Andy Dai committed Oct 4, 2024 at 14:58 UTC 0cc566ca8fd2d21a94f3a8e48bf5c5b60d42b59f
1 file changed +3 -1
benchmarks/benchmark_prefix_caching.py
+3 -1
@@ -113,7 +113,7 @@ def repeat_and_sort_requests(requests: List[Tuple[str, int, int]],
113 def main(args):
114 tokenizer = get_tokenizer(args.model, trust_remote_code=True)
115 input_length_range = tuple(map(int, args.input_length_range.split(':')))
116 -
116 + random.seed(args.seed)
117 if args.dataset_path is not None:
118 print(f"Start to sample {args.num_prompts} prompts"
119 "from {args.dataset_path}")
@@ -194,5 +194,7 @@ if __name__ == "__main__":
194 default='128:256',
195 help='Range of input lengths for sampling prompts,'
196 'specified as "min:max" (e.g., "128:256").')
197 + parser.add_argument("--seed", type=int, default=0,
198 + help='Random seed for reproducibility')
199 args = parser.parse_args()
200 main(args)