A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Minor fix in prefill cache example (#2494)
Jason Zhu committed
Jan 18, 2024 at 09:40 UTC
5d80a9178b48f211a6fa02a7b5ddc0a0ae29aa44
1 file changed
+10
-2
examples/offline_inference_with_prefix.py
+10
-2
@@ -40,8 +40,16 @@ print("-" * 80)
40
# -1 since the last token can change when concatenating prompts.
41
prefix_pos = len(llm.llm_engine.tokenizer.encode(prefix)) - 1
42
43
-# Generate with prefix
44
-outputs = llm.generate(generating_prompts, sampling_params,
43
+# The llm.generate call will batch all prompts and send the batch at once if resources allow.
44
+# The prefix will only be cached after the first batch is processed, so we need to call generate once
45
+# to calculate the prefix and cache it.
46
+outputs = llm.generate(generating_prompts[0],
47
+ sampling_params,
48
+ prefix_pos=[prefix_pos])
49
+
50
+# Subsequent batches can leverage the cached prefix
51
+outputs = llm.generate(generating_prompts,
52
+ sampling_params,
53
prefix_pos=[prefix_pos] * len(generating_prompts))
54
55
# Print the outputs. You should see the same outputs as before