@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d65049daab
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] Add random_seed to sample_hf_requests in benchmark_serving script (#9013)

Co-authored-by: Isotr0py <2037008807@qq.com>

Kai Wu committed Oct 17, 2024 at 14:11 UTC d65049daabe9a80783b0547fd85dd39a18a905b3
1 file changed +4 -2
benchmarks/benchmark_serving.py
+4 -2
@@ -202,6 +202,7 @@ def sample_hf_requests(
202 dataset_split: str,
203 num_requests: int,
204 tokenizer: PreTrainedTokenizerBase,
205 + random_seed: int,
206 fixed_output_len: Optional[int] = None,
207 ) -> List[Tuple[str, str, int, Optional[Dict[str, Collection[str]]]]]:
208 dataset = load_dataset(dataset_path,
@@ -210,8 +211,8 @@ def sample_hf_requests(
211 streaming=True)
212 assert "conversations" in dataset.features, (
213 "HF Dataset must have 'conversations' column.")
213 - filtered_dataset = dataset.shuffle().filter(
214 - lambda x: len(x["conversations"]) >= 2)
214 + filter_func = lambda x: len(x["conversations"]) >= 2
215 + filtered_dataset = dataset.shuffle(seed=random_seed).filter(filter_func)
216 sampled_requests: List[Tuple[str, int, int, Dict[str,
217 Collection[str]]]] = []
218 for data in filtered_dataset:
@@ -646,6 +647,7 @@ def main(args: argparse.Namespace):
647 dataset_split=args.hf_split,
648 num_requests=args.num_prompts,
649 tokenizer=tokenizer,
650 + random_seed=args.seed,
651 fixed_output_len=args.hf_output_len,
652 )
653