A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Misc] Improved prefix cache example (#9077)
Andy Dai committed
Oct 4, 2024 at 14:38 UTC
05c531be476e8a864a1ab83a65f7e056315ea1fc
1 file changed
+3
-9
examples/offline_inference_with_prefix.py
+3
-9
@@ -1,7 +1,8 @@
1
-from time import time
2
-
1
from vllm import LLM, SamplingParams
2
3
+# NOTE: This is just a running example. For benchmarking purpose,
4
+# please see benchmarks/benchmark_prefix_caching.py
5
+
6
# Common prefix.
7
prefix = (
8
"You are an expert school principal, skilled in effectively managing "
@@ -37,9 +38,7 @@ print("Results without `enable_prefix_caching`")
38
39
# Generate texts from the prompts. The output is a list of RequestOutput objects
40
# that contain the prompt, generated text, and other information.
40
-start_time_regular = time()
41
outputs = regular_llm.generate(generating_prompts, sampling_params)
42
-duration_regular = time() - start_time_regular
42
43
regular_generated_texts = []
44
# Print the outputs.
@@ -55,9 +54,7 @@ print("-" * 80)
54
prefix_cached_llm.generate(generating_prompts[0], sampling_params)
55
56
# Generate with prefix caching.
58
-start_time_cached = time()
57
outputs = prefix_cached_llm.generate(generating_prompts, sampling_params)
60
-duration_cached = time() - start_time_cached
58
59
print("Results with `enable_prefix_caching`")
60
@@ -77,6 +74,3 @@ generated_same = all([
74
for i in range(len(prompts))
75
])
76
print(f"Generated answers are the same: {generated_same}")
80
-
81
-speedup = round(duration_regular / duration_cached, 2)
82
-print(f"Speed up of cached generation compared to the regular is: {speedup}")