A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
[Benchmark] Fix `Value of type "SampleRequest" is not indexable` (#18032)
Signed-off-by: Brayden Zhong <b8zhong@uwaterloo.ca>
Brayden Zhong committed
Jun 20, 2025 at 00:28 UTC
5aa4a015ce4c85ad292a2f7d61df60a57ffc75b2
2 files changed
+2
-2
benchmarks/benchmark_throughput.py
+1
-1
@@ -97,7 +97,7 @@ def run_vllm(
97
assert lora_requests is None, "BeamSearch API does not support LoRA"
98
prompts = [request.prompt for request in requests]
99
# output_len should be the same for all requests.
100
- output_len = requests[0][2]
100
+ output_len = requests[0].expected_output_len
101
for request in requests:
102
assert request.expected_output_len == output_len
103
start = time.perf_counter()
vllm/benchmarks/throughput.py
+1
-1
@@ -84,7 +84,7 @@ def run_vllm(
84
assert lora_requests is None, "BeamSearch API does not support LoRA"
85
prompts = [request.prompt for request in requests]
86
# output_len should be the same for all requests.
87
- output_len = requests[0][2]
87
+ output_len = requests[0].expected_output_len
88
for request in requests:
89
assert request.expected_output_len == output_len
90
start = time.perf_counter()