A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
Update benchmark_serving.py to read and write json-datasets, results in UTF8, for better compatibility with Windows (#8997)
vlsav committed
Oct 1, 2024 at 21:07 UTC
22f5851b807376a836eb3551903c7fc6c81eaa9b
1 file changed
+3
-3
benchmarks/benchmark_serving.py
+3
-3
@@ -90,7 +90,7 @@ def sample_sharegpt_requests(
90
fixed_output_len: Optional[int] = None,
91
) -> List[Tuple[str, int, int, None]]:
92
# Load the dataset.
93
- with open(dataset_path) as f:
93
+ with open(dataset_path, encoding='utf-8') as f:
94
dataset = json.load(f)
95
# Filter out the conversations with less than 2 turns.
96
dataset = [data for data in dataset if len(data["conversations"]) >= 2]
@@ -139,7 +139,7 @@ def sample_sonnet_requests(
139
), "'args.sonnet-input-len' must be greater than 'args.prefix-input-len'."
140
141
# Load the dataset.
142
- with open(dataset_path) as f:
142
+ with open(dataset_path, encoding='utf-8') as f:
143
poem_lines = f.readlines()
144
145
# Tokenize the poem lines.
@@ -726,7 +726,7 @@ def main(args: argparse.Namespace):
726
file_name = args.result_filename
727
if args.result_dir:
728
file_name = os.path.join(args.result_dir, file_name)
729
- with open(file_name, "w") as outfile:
729
+ with open(file_name, "w", encoding='utf-8') as outfile:
730
json.dump(result_json, outfile)
731
732