@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ae8b633ba3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix] Fix offline_inference_with_prefix.py (#9505)

Tyler Michael Smith committed Oct 18, 2024 at 12:59 UTC ae8b633ba354eaad163e8decf0e4752b5ce58ac2
1 file changed +4 -2
examples/offline_inference_with_prefix.py
+4 -2
@@ -29,11 +29,13 @@ generating_prompts = [prefix + prompt for prompt in prompts]
29 sampling_params = SamplingParams(temperature=0.0)
30
31 # Create an LLM.
32 -regular_llm = LLM(model="facebook/opt-125m", gpu_memory_utilization=0.4)
32 +regular_llm = LLM(model="facebook/opt-125m", gpu_memory_utilization=0.3)
33
34 +# The second LLM needs to request a higher gpu_memory_utilization because
35 +# the first LLM has already allocated a full 30% of the gpu memory.
36 prefix_cached_llm = LLM(model="facebook/opt-125m",
37 enable_prefix_caching=True,
36 - gpu_memory_utilization=0.4)
38 + gpu_memory_utilization=0.6)
39 print("Results without `enable_prefix_caching`")
40
41 # Generate texts from the prompts. The output is a list of RequestOutput objects