A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
Disable prefix cache by default for benchmark (#18639)
Signed-off-by: cascade812 <cascade812@outlook.com>
cascade committed
May 27, 2025 at 05:06 UTC
aaa4ac1c95aaf70afab51582c56d80554a21bbd0
2 files changed
+6
benchmarks/benchmark_latency.py
+3
@@ -189,5 +189,8 @@ if __name__ == "__main__":
189
)
190
191
parser = EngineArgs.add_cli_args(parser)
192
+ # V1 enables prefix caching by default which skews the latency
193
+ # numbers. We need to disable prefix caching by default.
194
+ parser.set_defaults(enable_prefix_caching=False)
195
args = parser.parse_args()
196
main(args)
vllm/benchmarks/latency.py
+3
@@ -80,6 +80,9 @@ def add_cli_args(parser: argparse.ArgumentParser):
80
)
81
82
parser = EngineArgs.add_cli_args(parser)
83
+ # V1 enables prefix caching by default which skews the latency
84
+ # numbers. We need to disable prefix caching by default.
85
+ parser.set_defaults(enable_prefix_caching=True)
86
87
88
def main(args: argparse.Namespace):