A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Bugfix] Offline example of disaggregated prefill (#13214)
XiaobingZhang committed
Feb 14, 2025 at 12:20 UTC
84683fa27194dffb03831decb5f7f24abc5b12ba
1 file changed
+1
-1
examples/offline_inference/disaggregated_prefill.py
+1
-1
@@ -22,7 +22,7 @@ def run_prefill(prefill_done):
22
# and 3 and do prefilling on request 2.
23
prompts = [
24
"Hello, my name is",
25
- # "Hi, your name is",
25
+ "Hi, your name is",
26
# The decode node will actually "prefill" this request.
27
"Tell me a very long story",
28
]