@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c2d6d2f960
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Bugfix]: Fix issues related to prefix caching example (#5177) (#5180)

Daniil Arapov committed Jun 2, 2024 at 01:53 UTC c2d6d2f960176491e0499656409f30b947ee8027
1 file changed +37 -10
examples/offline_inference_with_prefix.py
+37 -10
@@ -1,5 +1,8 @@
1 +from time import time
2 +
3 from vllm import LLM, SamplingParams
4
5 +# Common prefix.
6 prefix = (
7 "You are an expert school principal, skilled in effectively managing "
8 "faculty and staff. Draft 10-15 questions for a potential first grade "
@@ -18,36 +21,60 @@ prompts = [
21 "The capital of France is",
22 "The future of AI is",
23 ]
24 +
25 +generating_prompts = [prefix + prompt for prompt in prompts]
26 +
27 # Create a sampling params object.
28 sampling_params = SamplingParams(temperature=0.0)
29
30 # Create an LLM.
25 -llm = LLM(model="facebook/opt-125m", enable_prefix_caching=True)
31 +regular_llm = LLM(model="facebook/opt-125m", gpu_memory_utilization=0.4)
32
27 -generating_prompts = [prefix + prompt for prompt in prompts]
33 +prefix_cached_llm = LLM(model="facebook/opt-125m",
34 + enable_prefix_caching=True,
35 + gpu_memory_utilization=0.4)
36 +print("Results without `enable_prefix_caching`")
37
38 # Generate texts from the prompts. The output is a list of RequestOutput objects
39 # that contain the prompt, generated text, and other information.
31 -outputs = llm.generate(generating_prompts, sampling_params)
40 +start_time_regular = time()
41 +outputs = regular_llm.generate(generating_prompts, sampling_params)
42 +duration_regular = time() - start_time_regular
43 +
44 +regular_generated_texts = []
45 # Print the outputs.
46 for output in outputs:
47 prompt = output.prompt
48 generated_text = output.outputs[0].text
49 + regular_generated_texts.append(generated_text)
50 print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
51
52 print("-" * 80)
53
54 # The llm.generate call will batch all prompts and send the batch at once
41 -# if resources allow. The prefix will only be cached after the first batch
42 -# is processed, so we need to call generate once to calculate the prefix
43 -# and cache it.
44 -outputs = llm.generate(generating_prompts[0], sampling_params)
55 +# if resources allow.
56 +start_time_cached = time()
57 +outputs = prefix_cached_llm.generate(generating_prompts, sampling_params)
58 +duration_cached = time() - start_time_cached
59
46 -# Subsequent batches can leverage the cached prefix
47 -outputs = llm.generate(generating_prompts, sampling_params)
60 +print("Results with `enable_prefix_caching`")
61
49 -# Print the outputs. You should see the same outputs as before
62 +cached_generated_texts = []
63 +# Print the outputs. You should see the same outputs as before.
64 for output in outputs:
65 prompt = output.prompt
66 generated_text = output.outputs[0].text
67 + cached_generated_texts.append(generated_text)
68 print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
69 +
70 +print("-" * 80)
71 +
72 +# Compare the results and display the speedup
73 +generated_same = all([
74 + regular_generated_texts[i] == cached_generated_texts[i]
75 + for i in range(len(prompts))
76 +])
77 +print(f"Generated answers are the same: {generated_same}")
78 +
79 +speedup = round(duration_regular / duration_cached, 2)
80 +print(f"Speed up of cached generation compared to the regular is: {speedup}")