A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Fix offline_inference_with_prefix.py (#9505)
Tyler Michael Smith committed
Oct 18, 2024 at 12:59 UTC
ae8b633ba354eaad163e8decf0e4752b5ce58ac2
1 file changed
+4
-2
examples/offline_inference_with_prefix.py
+4
-2
@@ -29,11 +29,13 @@ generating_prompts = [prefix + prompt for prompt in prompts]
29
sampling_params = SamplingParams(temperature=0.0)
30
31
# Create an LLM.
32
-regular_llm = LLM(model="facebook/opt-125m", gpu_memory_utilization=0.4)
32
+regular_llm = LLM(model="facebook/opt-125m", gpu_memory_utilization=0.3)
33
34
+# The second LLM needs to request a higher gpu_memory_utilization because
35
+# the first LLM has already allocated a full 30% of the gpu memory.
36
prefix_cached_llm = LLM(model="facebook/opt-125m",
37
enable_prefix_caching=True,
36
- gpu_memory_utilization=0.4)
38
+ gpu_memory_utilization=0.6)
39
print("Results without `enable_prefix_caching`")
40
41
# Generate texts from the prompts. The output is a list of RequestOutput objects