@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 22f5851b80
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

Update benchmark_serving.py to read and write json-datasets, results in UTF8, for better compatibility with Windows (#8997)

vlsav committed Oct 1, 2024 at 21:07 UTC 22f5851b807376a836eb3551903c7fc6c81eaa9b
1 file changed +3 -3
benchmarks/benchmark_serving.py
+3 -3
@@ -90,7 +90,7 @@ def sample_sharegpt_requests(
90 fixed_output_len: Optional[int] = None,
91 ) -> List[Tuple[str, int, int, None]]:
92 # Load the dataset.
93 - with open(dataset_path) as f:
93 + with open(dataset_path, encoding='utf-8') as f:
94 dataset = json.load(f)
95 # Filter out the conversations with less than 2 turns.
96 dataset = [data for data in dataset if len(data["conversations"]) >= 2]
@@ -139,7 +139,7 @@ def sample_sonnet_requests(
139 ), "'args.sonnet-input-len' must be greater than 'args.prefix-input-len'."
140
141 # Load the dataset.
142 - with open(dataset_path) as f:
142 + with open(dataset_path, encoding='utf-8') as f:
143 poem_lines = f.readlines()
144
145 # Tokenize the poem lines.
@@ -726,7 +726,7 @@ def main(args: argparse.Namespace):
726 file_name = args.result_filename
727 if args.result_dir:
728 file_name = os.path.join(args.result_dir, file_name)
729 - with open(file_name, "w") as outfile:
729 + with open(file_name, "w", encoding='utf-8') as outfile:
730 json.dump(result_json, outfile)
731
732