@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 05c531be47
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Misc] Improved prefix cache example (#9077)

Andy Dai committed Oct 4, 2024 at 14:38 UTC 05c531be476e8a864a1ab83a65f7e056315ea1fc
1 file changed +3 -9
examples/offline_inference_with_prefix.py
+3 -9
@@ -1,7 +1,8 @@
1 -from time import time
2 -
1 from vllm import LLM, SamplingParams
2
3 +# NOTE: This is just a running example. For benchmarking purpose,
4 +# please see benchmarks/benchmark_prefix_caching.py
5 +
6 # Common prefix.
7 prefix = (
8 "You are an expert school principal, skilled in effectively managing "
@@ -37,9 +38,7 @@ print("Results without `enable_prefix_caching`")
38
39 # Generate texts from the prompts. The output is a list of RequestOutput objects
40 # that contain the prompt, generated text, and other information.
40 -start_time_regular = time()
41 outputs = regular_llm.generate(generating_prompts, sampling_params)
42 -duration_regular = time() - start_time_regular
42
43 regular_generated_texts = []
44 # Print the outputs.
@@ -55,9 +54,7 @@ print("-" * 80)
54 prefix_cached_llm.generate(generating_prompts[0], sampling_params)
55
56 # Generate with prefix caching.
58 -start_time_cached = time()
57 outputs = prefix_cached_llm.generate(generating_prompts, sampling_params)
60 -duration_cached = time() - start_time_cached
58
59 print("Results with `enable_prefix_caching`")
60
@@ -77,6 +74,3 @@ generated_same = all([
74 for i in range(len(prompts))
75 ])
76 print(f"Generated answers are the same: {generated_same}")
80 -
81 -speedup = round(duration_regular / duration_cached, 2)
82 -print(f"Speed up of cached generation compared to the regular is: {speedup}")