A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] Set enable_prefix_caching=True in prefix caching example (#3703)
Woosuk Kwon committed
Mar 28, 2024 at 16:26 UTC
c0935c96d3472fe153ec849311c78fd8728dbb3c
1 file changed
+1
-1
examples/offline_inference_with_prefix.py
+1
-1
@@ -22,7 +22,7 @@ prompts = [
22
sampling_params = SamplingParams(temperature=0.0)
23
24
# Create an LLM.
25
-llm = LLM(model="facebook/opt-125m")
25
+llm = LLM(model="facebook/opt-125m", enable_prefix_caching=True)
26
27
generating_prompts = [prefix + prompt for prompt in prompts]
28