@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e585b583a9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Bugfix] Support testing prefill throughput with benchmark_serving.py --hf-output-len 1 (#8891)

Chen Zhang committed Sep 28, 2024 at 11:51 UTC e585b583a92903c9a5cc8055a444a208f4387891
1 file changed +4 -5
benchmarks/benchmark_serving.py
+4 -5
@@ -89,8 +89,6 @@ def sample_sharegpt_requests(
89 tokenizer: PreTrainedTokenizerBase,
90 fixed_output_len: Optional[int] = None,
91 ) -> List[Tuple[str, int, int, None]]:
92 - if fixed_output_len is not None and fixed_output_len < 4:
93 - raise ValueError("output_len too small")
92 # Load the dataset.
93 with open(dataset_path) as f:
94 dataset = json.load(f)
@@ -117,7 +115,7 @@ def sample_sharegpt_requests(
115 prompt_len = len(prompt_token_ids)
116 output_len = len(completion_token_ids
117 ) if fixed_output_len is None else fixed_output_len
120 - if prompt_len < 4 or output_len < 4:
118 + if prompt_len < 4 or (fixed_output_len is None and output_len < 4):
119 # Prune too short sequences.
120 continue
121 if prompt_len > 1024 or prompt_len + output_len > 2048:
@@ -228,10 +226,11 @@ def sample_hf_requests(
226 prompt_len = len(prompt_token_ids)
227 output_len = len(completion_token_ids
228 ) if fixed_output_len is None else fixed_output_len
231 - if prompt_len < 4 or output_len < 4:
229 + if fixed_output_len is None and (prompt_len < 4 or output_len < 4):
230 # Prune too short sequences.
231 continue
234 - if prompt_len > 1024 or prompt_len + output_len > 2048:
232 + if fixed_output_len is None and \
233 + (prompt_len > 1024 or prompt_len + output_len > 2048):
234 # Prune too long sequences.
235 continue
236