@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c0935c96d3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] Set enable_prefix_caching=True in prefix caching example (#3703)

Woosuk Kwon committed Mar 28, 2024 at 16:26 UTC c0935c96d3472fe153ec849311c78fd8728dbb3c
1 file changed +1 -1
examples/offline_inference_with_prefix.py
+1 -1
@@ -22,7 +22,7 @@ prompts = [
22 sampling_params = SamplingParams(temperature=0.0)
23
24 # Create an LLM.
25 -llm = LLM(model="facebook/opt-125m")
25 +llm = LLM(model="facebook/opt-125m", enable_prefix_caching=True)
26
27 generating_prompts = [prefix + prompt for prompt in prompts]
28