A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 21:15 UTC
[ROCm][P/D][MORI][BugFix] Ensure correct api is used when making requests to prefill / decode nodes (#39835)
Signed-off-by: Randall Smith <Randall.Smith@amd.com>
rasmith committed
Apr 21, 2026 at 19:48 UTC
cefa5281a752068aed17208506054b03322e4d37
2 files changed
+28
-13
examples/online_serving/disaggregated_serving/moriio_toy_proxy_server.py
+27
-12
@@ -12,7 +12,7 @@ import aiohttp
12
import msgpack
13
import regex as re
14
import zmq
15
-from quart import Quart, make_response, request
15
+from quart import Quart, Request, make_response, request
16
17
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
18
MoRIIOConstants,
@@ -139,10 +139,13 @@ async def send_request_to_prefill(
139
return await response.json()
140
141
else:
142
- raise RuntimeError(
143
- "send_request_to_prefill response.status != 200response.status = ",
144
- response.status,
142
+ error_message = (
143
+ f"send_request_to_prefill response ={response},"
144
+ f"reason={response.reason}, status={response.status},"
145
+ f"method={response.method}, url={response.url},"
146
+ f"real_url={response.real_url}"
147
)
148
+ raise RuntimeError(error_message)
149
150
151
async def start_decode_request(endpoint, req_data, request_id):
@@ -163,9 +166,13 @@ async def stream_decode_response(session, response, request_id):
166
async for chunk_bytes in response.content.iter_chunked(1024):
167
yield chunk_bytes
168
else:
166
- raise RuntimeError(
167
- f"decode response.status != 200, status = {response.status}"
169
+ error_message = (
170
+ f"stream_decode_response response ={response},"
171
+ f"reason={response.reason}, status={response.status},"
172
+ f"method={response.method}, url={response.url},"
173
+ f"real_url={response.real_url}"
174
)
175
+ raise RuntimeError(error_message)
176
finally:
177
await session.close()
178
@@ -175,8 +182,16 @@ def example_round_robin_dp_loader(request_number, dp_size):
182
183
184
@app.route("/v1/completions", methods=["POST"])
185
+async def handle_completions_request():
186
+ return await handle_request("/completions", request)
187
+
188
+
189
@app.route("/v1/chat/completions", methods=["POST"])
179
-async def handle_request():
190
+async def handle_chat_completions_request():
191
+ return await handle_request("/chat/completions", request)
192
+
193
+
194
+async def handle_request(api: str, request: Request):
195
try:
196
with _list_lock:
197
global request_nums
@@ -230,9 +245,10 @@ async def handle_request():
245
)
246
req_data_to_prefill["kv_transfer_params"]["transfer_id"] = transfer_id
247
248
+ prefill_request_url = prefill_instance_endpoint["request_address"] + api
249
send_prefill_task = asyncio.create_task(
250
send_request_to_prefill(
235
- prefill_instance_endpoint["request_address"],
251
+ prefill_request_url,
252
req_data_to_prefill,
253
request_id,
254
decode_instance_endpoint,
@@ -241,7 +257,7 @@ async def handle_request():
257
selected_prefill_dp_rank,
258
)
259
)
244
- ip, port = extract_ip_port_fast(prefill_instance_endpoint["request_address"])
260
+ ip, port = extract_ip_port_fast(prefill_request_url)
261
262
req_data["max_tokens"] -= 1
263
@@ -276,10 +292,9 @@ async def handle_request():
292
req_data["kv_transfer_params"]["remote_dp_rank"] = selected_prefill_dp_rank
293
req_data["kv_transfer_params"]["transfer_id"] = transfer_id
294
295
+ decode_request_url = decode_instance_endpoint["request_address"] + api
296
decode_request_task = asyncio.create_task(
280
- start_decode_request(
281
- decode_instance_endpoint["request_address"], req_data, request_id
282
- )
297
+ start_decode_request(decode_request_url, req_data, request_id)
298
)
299
300
session, decode_response = await decode_request_task
vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py
+1
-1
@@ -846,7 +846,7 @@ class MoRIIOConnectorWorker:
846
]
847
848
def _ping(self, zmq_context):
849
- http_request_address = f"http://{self.request_address}/v1/completions"
849
+ http_request_address = f"http://{self.request_address}/v1"
850
role = "P" if self.is_producer else "D"
851
852
retry_count = 0