A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
Fix `benchmark_throughput.py --backend=hf` (#16352)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Apr 9, 2025 at 13:09 UTC
b2ce859bd28b6a845962cba575ab08b74011617b
1 file changed
+5
-2
benchmarks/benchmark_throughput.py
+5
-2
@@ -213,14 +213,17 @@ def run_hf(
213
max_prompt_len = 0
214
max_output_len = 0
215
for i in range(len(requests)):
216
- prompt, prompt_len, output_len = requests[i]
216
+ prompt = requests[i].prompt
217
+ prompt_len = requests[i].prompt_len
218
+ output_len = requests[i].expected_output_len
219
# Add the prompt to the batch.
220
batch.append(prompt)
221
max_prompt_len = max(max_prompt_len, prompt_len)
222
max_output_len = max(max_output_len, output_len)
223
if len(batch) < max_batch_size and i != len(requests) - 1:
224
# Check if we can add more requests to the batch.
223
- _, next_prompt_len, next_output_len = requests[i + 1]
225
+ next_prompt_len = requests[i + 1].prompt_len
226
+ next_output_len = requests[i + 1].expected_output_len
227
if (max(max_prompt_len, next_prompt_len) +
228
max(max_output_len, next_output_len)) <= 2048:
229
# We can add more requests to the batch.