A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
[Minor] Fix benchmark_throughput.py (#151)
Woosuk Kwon committed
Jun 16, 2023 at 21:00 UTC
bab8f3dd0ddf18ed0e28f77de57f0e55c7097aff
1 file changed
+1
benchmarks/benchmark_throughput.py
+1
@@ -97,6 +97,7 @@ def run_cacheflow(
97
# FIXME(woosuk): Do not use internal method.
98
llm._add_request(
99
prompt=prompt,
100
+ prompt_token_ids=None,
101
sampling_params=sampling_params,
102
)
103