A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Bugfix] Fix a path bug in disaggregated prefill example script. (#12121)
Signed-off-by: Kuntai Du <kuntai@uchicago.edu>
Kuntai Du committed
Jan 17, 2025 at 11:12 UTC
ebc73f2828df48f0ffbb99e52f0e4b394a23dbd3
1 file changed
+3
-1
examples/online_serving/disaggregated_prefill.sh
+3
-1
@@ -3,6 +3,8 @@
3
# We will launch 2 vllm instances (1 for prefill and 1 for decode),
4
# and then transfer the KV cache between them.
5
6
+set -xe
7
+
8
echo "🚧🚧 Warning: The usage of disaggregated prefill is experimental and subject to change 🚧🚧"
9
sleep 1
10
@@ -69,7 +71,7 @@ wait_for_server 8200
71
# instance
72
# NOTE: the usage of this API is subject to change --- in the future we will
73
# introduce "vllm connect" to connect between prefill and decode instances
72
-python3 ../benchmarks/disagg_benchmarks/disagg_prefill_proxy_server.py &
74
+python3 ../../benchmarks/disagg_benchmarks/disagg_prefill_proxy_server.py &
75
sleep 1
76
77
# serve two example requests