A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Bugfix]: Fix issues related to prefix caching example (#5177) (#5180)
Daniil Arapov committed
Jun 2, 2024 at 01:53 UTC
c2d6d2f960176491e0499656409f30b947ee8027
1 file changed
+37
-10
examples/offline_inference_with_prefix.py
+37
-10
@@ -1,5 +1,8 @@
1
+from time import time
2
+
3
from vllm import LLM, SamplingParams
4
5
+# Common prefix.
6
prefix = (
7
"You are an expert school principal, skilled in effectively managing "
8
"faculty and staff. Draft 10-15 questions for a potential first grade "
@@ -18,36 +21,60 @@ prompts = [
21
"The capital of France is",
22
"The future of AI is",
23
]
24
+
25
+generating_prompts = [prefix + prompt for prompt in prompts]
26
+
27
# Create a sampling params object.
28
sampling_params = SamplingParams(temperature=0.0)
29
30
# Create an LLM.
25
-llm = LLM(model="facebook/opt-125m", enable_prefix_caching=True)
31
+regular_llm = LLM(model="facebook/opt-125m", gpu_memory_utilization=0.4)
32
27
-generating_prompts = [prefix + prompt for prompt in prompts]
33
+prefix_cached_llm = LLM(model="facebook/opt-125m",
34
+ enable_prefix_caching=True,
35
+ gpu_memory_utilization=0.4)
36
+print("Results without `enable_prefix_caching`")
37
38
# Generate texts from the prompts. The output is a list of RequestOutput objects
39
# that contain the prompt, generated text, and other information.
31
-outputs = llm.generate(generating_prompts, sampling_params)
40
+start_time_regular = time()
41
+outputs = regular_llm.generate(generating_prompts, sampling_params)
42
+duration_regular = time() - start_time_regular
43
+
44
+regular_generated_texts = []
45
# Print the outputs.
46
for output in outputs:
47
prompt = output.prompt
48
generated_text = output.outputs[0].text
49
+ regular_generated_texts.append(generated_text)
50
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
51
52
print("-" * 80)
53
54
# The llm.generate call will batch all prompts and send the batch at once
41
-# if resources allow. The prefix will only be cached after the first batch
42
-# is processed, so we need to call generate once to calculate the prefix
43
-# and cache it.
44
-outputs = llm.generate(generating_prompts[0], sampling_params)
55
+# if resources allow.
56
+start_time_cached = time()
57
+outputs = prefix_cached_llm.generate(generating_prompts, sampling_params)
58
+duration_cached = time() - start_time_cached
59
46
-# Subsequent batches can leverage the cached prefix
47
-outputs = llm.generate(generating_prompts, sampling_params)
60
+print("Results with `enable_prefix_caching`")
61
49
-# Print the outputs. You should see the same outputs as before
62
+cached_generated_texts = []
63
+# Print the outputs. You should see the same outputs as before.
64
for output in outputs:
65
prompt = output.prompt
66
generated_text = output.outputs[0].text
67
+ cached_generated_texts.append(generated_text)
68
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
69
+
70
+print("-" * 80)
71
+
72
+# Compare the results and display the speedup
73
+generated_same = all([
74
+ regular_generated_texts[i] == cached_generated_texts[i]
75
+ for i in range(len(prompts))
76
+])
77
+print(f"Generated answers are the same: {generated_same}")
78
+
79
+speedup = round(duration_regular / duration_cached, 2)
80
+print(f"Speed up of cached generation compared to the regular is: {speedup}")