A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Add random_seed to sample_hf_requests in benchmark_serving script (#9013)
Co-authored-by: Isotr0py <2037008807@qq.com>
Kai Wu committed
Oct 17, 2024 at 14:11 UTC
d65049daabe9a80783b0547fd85dd39a18a905b3
1 file changed
+4
-2
benchmarks/benchmark_serving.py
+4
-2
@@ -202,6 +202,7 @@ def sample_hf_requests(
202
dataset_split: str,
203
num_requests: int,
204
tokenizer: PreTrainedTokenizerBase,
205
+ random_seed: int,
206
fixed_output_len: Optional[int] = None,
207
) -> List[Tuple[str, str, int, Optional[Dict[str, Collection[str]]]]]:
208
dataset = load_dataset(dataset_path,
@@ -210,8 +211,8 @@ def sample_hf_requests(
211
streaming=True)
212
assert "conversations" in dataset.features, (
213
"HF Dataset must have 'conversations' column.")
213
- filtered_dataset = dataset.shuffle().filter(
214
- lambda x: len(x["conversations"]) >= 2)
214
+ filter_func = lambda x: len(x["conversations"]) >= 2
215
+ filtered_dataset = dataset.shuffle(seed=random_seed).filter(filter_func)
216
sampled_requests: List[Tuple[str, int, int, Dict[str,
217
Collection[str]]]] = []
218
for data in filtered_dataset:
@@ -646,6 +647,7 @@ def main(args: argparse.Namespace):
647
dataset_split=args.hf_split,
648
num_requests=args.num_prompts,
649
tokenizer=tokenizer,
650
+ random_seed=args.seed,
651
fixed_output_len=args.hf_output_len,
652
)
653