A high-throughput and memory-efficient inference and serving engine for LLMs
[Bench] benchmark_serving_multi_turn: make non-standard conversation_id payload opt-in (#43756)
Signed-off-by: Change72 <cguo51@asu.edu>
Change72 committed
Jun 9, 2026 at 19:41 UTC
320c52b1342ad961091bb3333b867c0899907b06
3 files changed
+61
-2
benchmarks/multi_turn/benchmark_serving_multi_turn.py
+19
-1
@@ -65,6 +65,7 @@ class RequestArgs(NamedTuple):
65
limit_min_tokens: int # Use negative value for no limit
66
limit_max_tokens: int # Use negative value for no limit
67
timeout_sec: int
68
+ send_conversation_id: bool
69
headers: dict[str, str]
70
71
@@ -453,7 +454,7 @@ async def send_turn(
454
min_tokens,
455
max_tokens,
456
req_args.timeout_sec,
456
- conversation_id=conv_id,
457
+ conversation_id=conv_id if req_args.send_conversation_id else None,
458
headers=req_args.headers,
459
)
460
@@ -912,6 +913,7 @@ def get_client_config(
913
limit_min_tokens=args.limit_min_tokens,
914
limit_max_tokens=args.limit_max_tokens,
915
timeout_sec=args.request_timeout_sec,
916
+ send_conversation_id=args.send_conversation_id,
917
headers=headers,
918
)
919
@@ -1486,6 +1488,22 @@ async def main() -> None:
1488
help="Disable stream/streaming mode (set 'stream' to False in the API request)",
1489
)
1490
1491
+ parser.add_argument(
1492
+ "--send-conversation-id",
1493
+ default=False,
1494
+ action="store_true",
1495
+ help=(
1496
+ "Inject a `conversation_id` field into each Chat Completions "
1497
+ "payload. This is a non-standard OpenAI extension consumed by "
1498
+ "vLLM's disaggregated multi-turn proxy "
1499
+ "(examples/disaggregated/disaggregated_serving/"
1500
+ "disagg_proxy_multiturn.py) to key cross-turn KV cache reuse. "
1501
+ "Leave disabled (default) when targeting strict "
1502
+ "OpenAI-compatible endpoints; enable when benchmarking the "
1503
+ "disaggregated proxy."
1504
+ ),
1505
+ )
1506
+
1507
parser.add_argument(
1508
"-e",
1509
"--excel-output",
docs/features/nixl_connector_usage.md
+15
@@ -294,6 +294,21 @@ curl http://localhost:8000/v1/chat/completions \
294
!!! note
295
The `conversation_id` field is a non-standard extension to the OpenAI API. It is consumed by the proxy and not forwarded to the vLLM engine.
296
297
+### Benchmarking the multi-turn proxy
298
+
299
+[`benchmarks/multi_turn/benchmark_serving_multi_turn.py`](../../benchmarks/multi_turn/benchmark_serving_multi_turn.py) supports targeting the disaggregated multi-turn proxy with the `--send-conversation-id` flag, which injects a per-conversation `conversation_id` into every request payload so the proxy can key cross-turn KV cache reuse.
300
+
301
+The flag is **off by default** so the benchmark is compatible with strict OpenAI-compatible frontends that reject unknown top-level fields. When benchmarking the multi-turn proxy you must pass it explicitly — otherwise every turn lands as a cache MISS and the bidirectional KV transfer path is never exercised.
302
+
303
+```bash
304
+python benchmarks/multi_turn/benchmark_serving_multi_turn.py \
305
+ --model <MODEL> --served-model-name <NAME> \
306
+ --url http://<proxy_host>:8000 \
307
+ --input-file benchmarks/multi_turn/generate_multi_turn.json \
308
+ --num-clients 2 --max-active-conversations 6 \
309
+ --send-conversation-id
310
+```
311
+
312
### Limitations
313
314
- Requires a stateful proxy (or equivalent router) to track and forward `kv_transfer_params` between turns.
examples/disaggregated/disaggregated_serving/disagg_proxy_multiturn.py
+27
-1
@@ -35,12 +35,36 @@ Conversation isolation:
35
the JSON body) to scope the KV cache across turns. Without it, the
36
proxy cannot link turns and falls back to no-cache behavior.
37
38
+ ``conversation_id`` is a non-standard extension to the OpenAI Chat
39
+ Completions schema, consumed by this proxy and not forwarded to the
40
+ vLLM engine. Strict OpenAI-compatible frontends reject unknown
41
+ fields, so clients must opt in only when targeting this proxy.
42
+
43
Usage:
44
python disagg_proxy_multiturn.py \\
45
--host 0.0.0.0 --port 8000 \\
46
--prefiller-host 10.0.0.1 --prefiller-port 8100 \\
47
--decoder-host 10.0.0.2 --decoder-port 8200
48
49
+Benchmarking:
50
+ Use ``benchmarks/multi_turn/benchmark_serving_multi_turn.py`` with
51
+ the ``--send-conversation-id`` flag to inject a per-conversation
52
+ ``conversation_id`` into every request so this proxy can key
53
+ cross-turn KV cache reuse. The flag is *off by default*: without
54
+ it the benchmark sends OpenAI-schema-compliant payloads and every
55
+ turn lands as a cache MISS in this proxy.
56
+
57
+ Example:
58
+ python benchmarks/multi_turn/benchmark_serving_multi_turn.py \\
59
+ --model <MODEL> --served-model-name <NAME> \\
60
+ --url http://<proxy_host>:8000 \\
61
+ --input-file generate_multi_turn.json \\
62
+ --num-clients 2 --max-active-conversations 6 \\
63
+ --send-conversation-id
64
+
65
+ See ``docs/features/nixl_connector_usage.md`` for the broader
66
+ bidirectional-KV-transfer setup these benchmarks exercise.
67
+
68
Dependencies:
69
pip install fastapi uvicorn httpx
70
"""
@@ -373,7 +397,9 @@ async def _handle_request(api_path: str, request: Request):
397
logger.warning(
398
"[%s] No conversation_id provided — KV cache reuse disabled "
399
"for this request. Add a 'conversation_id' field to enable "
376
- "cross-turn KV sharing.",
400
+ "cross-turn KV sharing. When using "
401
+ "benchmarks/multi_turn/benchmark_serving_multi_turn.py, pass "
402
+ "--send-conversation-id (off by default).",
403
request_id,
404
)
405