A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Bugfix] Support testing prefill throughput with benchmark_serving.py --hf-output-len 1 (#8891)
Chen Zhang committed
Sep 28, 2024 at 11:51 UTC
e585b583a92903c9a5cc8055a444a208f4387891
1 file changed
+4
-5
benchmarks/benchmark_serving.py
+4
-5
@@ -89,8 +89,6 @@ def sample_sharegpt_requests(
89
tokenizer: PreTrainedTokenizerBase,
90
fixed_output_len: Optional[int] = None,
91
) -> List[Tuple[str, int, int, None]]:
92
- if fixed_output_len is not None and fixed_output_len < 4:
93
- raise ValueError("output_len too small")
92
# Load the dataset.
93
with open(dataset_path) as f:
94
dataset = json.load(f)
@@ -117,7 +115,7 @@ def sample_sharegpt_requests(
115
prompt_len = len(prompt_token_ids)
116
output_len = len(completion_token_ids
117
) if fixed_output_len is None else fixed_output_len
120
- if prompt_len < 4 or output_len < 4:
118
+ if prompt_len < 4 or (fixed_output_len is None and output_len < 4):
119
# Prune too short sequences.
120
continue
121
if prompt_len > 1024 or prompt_len + output_len > 2048:
@@ -228,10 +226,11 @@ def sample_hf_requests(
226
prompt_len = len(prompt_token_ids)
227
output_len = len(completion_token_ids
228
) if fixed_output_len is None else fixed_output_len
231
- if prompt_len < 4 or output_len < 4:
229
+ if fixed_output_len is None and (prompt_len < 4 or output_len < 4):
230
# Prune too short sequences.
231
continue
234
- if prompt_len > 1024 or prompt_len + output_len > 2048:
232
+ if fixed_output_len is None and \
233
+ (prompt_len > 1024 or prompt_len + output_len > 2048):
234
# Prune too long sequences.
235
continue
236