A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[PD] let p2p nccl toy proxy handle /chat/completions (#21734)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Jul 28, 2025 at 19:45 UTC
63fe3a700fd23aa7b2cb43cbd1cb7af960832603
1 file changed
+3
-2
examples/online_serving/disaggregated_serving_p2p_nccl_xpyd/disagg_proxy_p2p_nccl_xpyd.py
+3
-2
@@ -120,6 +120,7 @@ async def forward_request(url, data, request_id):
120
121
122
@app.route("/v1/completions", methods=["POST"])
123
+@app.route("/v1/chat/completions", methods=["POST"])
124
async def handle_request():
125
try:
126
original_request_data = await request.get_json()
@@ -157,13 +158,13 @@ async def handle_request():
158
159
# finish prefill
160
async for _ in forward_request(
160
- f"http://{prefill_addr}/v1/completions", prefill_request, request_id
161
+ f"http://{prefill_addr}{request.path}", prefill_request, request_id
162
):
163
continue
164
165
# return decode
166
generator = forward_request(
166
- f"http://{decode_addr}/v1/completions", original_request_data, request_id
167
+ f"http://{decode_addr}{request.path}", original_request_data, request_id
168
)
169
response = await make_response(generator)
170
response.timeout = None