A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 20:00 UTC
[Disagg] Support large batch size in proxy server and update NixlConnector doc for DP (#28782)
Signed-off-by: Ming Yang <minos.future@gmail.com>
Ming Yang committed
Dec 8, 2025 at 16:01 UTC
60d17251c920ae3c9d02e4b4101b738e4905aee4
3 files changed
+42
-4
docs/features/nixl_connector_usage.md
+2
@@ -146,6 +146,8 @@ python tests/v1/kv_connector/nixl_integration/toy_proxy_server.py \
146
--decoder-ports 8000 8000
147
```
148
149
+For multi-host DP deployment, only need to provide the host/port of the head instances.
150
+
151
### KV Role Options
152
153
- **kv_producer**: For prefiller instances that generate KV caches
examples/others/lmcache/disagg_prefill_lmcache_v1/disagg_proxy_server.py
+19
-2
@@ -26,9 +26,21 @@ async def lifespan(app: FastAPI):
26
)
27
28
app.state.prefill_client = httpx.AsyncClient(
29
- timeout=None, base_url=prefiller_base_url
29
+ timeout=None,
30
+ base_url=prefiller_base_url,
31
+ limits=httpx.Limits(
32
+ max_connections=None,
33
+ max_keepalive_connections=None,
34
+ ),
35
+ )
36
+ app.state.decode_client = httpx.AsyncClient(
37
+ timeout=None,
38
+ base_url=decoder_base_url,
39
+ limits=httpx.Limits(
40
+ max_connections=None,
41
+ max_keepalive_connections=None,
42
+ ),
43
)
31
- app.state.decode_client = httpx.AsyncClient(timeout=None, base_url=decoder_base_url)
44
45
yield
46
@@ -105,6 +117,11 @@ async def send_request_to_service(
117
headers = {"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"}
118
response = await client.post(endpoint, json=req_data, headers=headers)
119
response.raise_for_status()
120
+
121
+ # read/consume the response body to release the connection
122
+ # otherwise, it would http.ReadError
123
+ await response.aread()
124
+
125
return response
126
127
tests/v1/kv_connector/nixl_integration/toy_proxy_server.py
+21
-2
@@ -30,7 +30,14 @@ async def lifespan(app: FastAPI):
30
prefiller_base_url = f"http://{host}:{port}/v1"
31
app.state.prefill_clients.append(
32
{
33
- "client": httpx.AsyncClient(timeout=None, base_url=prefiller_base_url),
33
+ "client": httpx.AsyncClient(
34
+ timeout=None,
35
+ base_url=prefiller_base_url,
36
+ limits=httpx.Limits(
37
+ max_connections=None,
38
+ max_keepalive_connections=None,
39
+ ),
40
+ ),
41
"host": host,
42
"port": port,
43
"id": i,
@@ -42,7 +49,14 @@ async def lifespan(app: FastAPI):
49
decoder_base_url = f"http://{host}:{port}/v1"
50
app.state.decode_clients.append(
51
{
45
- "client": httpx.AsyncClient(timeout=None, base_url=decoder_base_url),
52
+ "client": httpx.AsyncClient(
53
+ timeout=None,
54
+ base_url=decoder_base_url,
55
+ limits=httpx.Limits(
56
+ max_connections=None,
57
+ max_keepalive_connections=None,
58
+ ),
59
+ ),
60
"host": host,
61
"port": port,
62
"id": i,
@@ -169,6 +183,10 @@ async def send_request_to_service(
183
)
184
response.raise_for_status()
185
186
+ # read/consume the response body to release the connection
187
+ # otherwise, it would http.ReadError
188
+ await response.aread()
189
+
190
return response
191
192
@@ -206,6 +224,7 @@ async def _handle_completions(api: str, request: Request):
224
225
# Extract the needed fields
226
response_json = response.json()
227
+ await response.aclose() # CRITICAL: Release connection back to pool
228
kv_transfer_params = response_json.get("kv_transfer_params", {})
229
if kv_transfer_params:
230
req_data["kv_transfer_params"] = kv_transfer_params