@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d98b8f371c

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 1, 21:15 UTC

[NixlConnector] Initiate deprecation cycle for `kv_both` role (#43874)

Signed-off-by: NickLucche <nlucches@redhat.com>

Nicolò Lucchesi committed Jun 5, 2026 at 11:08 UTC d98b8f371c503d4b86039d43e486116043b706b3

10 files changed +94 -24

docs/design/nixl_kv_cache_lease.md

+1 -1

@@ -128,7 +128,7 @@ The lease mechanism is controlled through `kv_connector_extra_config` in `--kv-t

128 vllm serve <MODEL> \

129 --kv-transfer-config '{

130 "kv_connector": "NixlConnector",

131 - "kv_role": "kv_both",

131 + "kv_role": "kv_producer",

132 "kv_connector_extra_config": {"kv_lease_duration": 60}

133 }'

134 ```

docs/features/nixl_connector_usage.md

+19 -9

@@ -50,7 +50,7 @@ To select a different backend, set `kv_connector_extra_config.backends` in `--kv

50 vllm serve <MODEL> \

51 --kv-transfer-config '{

52 "kv_connector":"NixlConnector",

53 - "kv_role":"kv_both",

53 + "kv_role":"kv_producer",

54 "kv_connector_extra_config":{"backends":["LIBFABRIC"]}

55 }'

56 ```

@@ -60,7 +60,7 @@ You can also pass JSON keys individually using dotted arguments, and you can app

60 ```bash

61 vllm serve <MODEL> \

62 --kv-transfer-config.kv_connector NixlConnector \

63 - --kv-transfer-config.kv_role kv_both \

63 + --kv-transfer-config.kv_role kv_producer \

64 --kv-transfer-config.kv_connector_extra_config.backends+ LIBFABRIC

65 ```

@@ -81,7 +81,7 @@ VLLM_NIXL_SIDE_CHANNEL_PORT=5600 \

81 vllm serve Qwen/Qwen3-0.6B \

82 --port 8100 \

83 --enforce-eager \

84 - --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail"}'

84 + --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_producer","kv_load_failure_policy":"fail"}'

85 ```

87 ### Consumer (Decoder) Configuration

@@ -96,7 +96,7 @@ VLLM_NIXL_SIDE_CHANNEL_PORT=5601 \

96 vllm serve Qwen/Qwen3-0.6B \

97 --port 8200 \

98 --enforce-eager \

99 - --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail"}'

99 + --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail"}'

100 ```

101

102 ### Proxy Server

@@ -212,10 +212,21 @@ sequenceDiagram

212 Enable bidirectional KV transfer by setting `bidirectional_kv_xfer` in `kv_connector_extra_config` on **both** P and D instances:

213

214 ```bash

215 +# Prefill instance

216 +vllm serve <MODEL> \

217 + --kv-transfer-config '{

218 + "kv_connector": "NixlConnector",

219 + "kv_role": "kv_producer",

220 + "kv_connector_extra_config": {

221 + "bidirectional_kv_xfer": true

222 + }

223 + }'

224 +

225 +# Decode instance

226 vllm serve <MODEL> \

227 --kv-transfer-config '{

228 "kv_connector": "NixlConnector",

218 - "kv_role": "kv_both",

229 + "kv_role": "kv_consumer",

230 "kv_connector_extra_config": {

231 "bidirectional_kv_xfer": true

232 }

@@ -359,11 +370,10 @@ For multi-host DP deployment, only need to provide the host/port of the head ins

370

371 - **kv_producer**: For prefiller instances that generate KV caches

372 - **kv_consumer**: For decoder instances that consume KV caches from prefiller

362 -- **kv_both**: Enables symmetric functionality where the connector can act as both producer and consumer. This provides flexibility for experimental setups and scenarios where the role distinction is not predetermined.

373 +- **kv_both** (deprecated): Previously used as a catch-all when the role was not predetermined. This value is now deprecated for NixlConnector and will be removed in a future release.

374

364 -!!! tip

365 - NixlConnector currently does not distinguish `kv_role`; the actual prefiller/decoder roles are determined by the upper-level proxy (e.g., `toy_proxy_server.py` using `--prefiller-hosts` and `--decoder-hosts`).

366 - Therefore, `kv_role` in `--kv-transfer-config` is effectively a placeholder and does not affect NixlConnector's behavior.

375 +!!! warning

376 + `kv_role="kv_both"` is deprecated for NixlConnector. Please set `kv_role="kv_producer"` for prefill instances and `kv_role="kv_consumer"` for decode instances. See [#33702](https://github.com/vllm-project/vllm/issues/33702) for details.

377

378 ### KV Load Failure Policy

379

tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh

+7 -5

@@ -49,11 +49,13 @@ else

49 KV_EXTRA_CONFIG=''

50 fi

52 -# Build the kv-transfer-config once

52 +# Build the kv-transfer-config for P and D

53 if [[ "$KV_BUFFER_DEVICE" == "cuda" ]]; then

54 - KV_CONFIG='{"kv_connector":"NixlConnector","kv_role":"kv_both"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}'

54 + KV_CONFIG_P='{"kv_connector":"NixlConnector","kv_role":"kv_producer"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}'

55 + KV_CONFIG_D='{"kv_connector":"NixlConnector","kv_role":"kv_consumer"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}'

56 else

56 - KV_CONFIG="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}"

57 + KV_CONFIG_P="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_producer\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}"

58 + KV_CONFIG_D="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_consumer\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}"

59 fi

61 # Models to run

@@ -159,7 +161,7 @@ run_tests_for_model() {

161 --block-size ${PREFILL_BLOCK_SIZE} \

162 --gpu-memory-utilization $GPU_MEMORY_UTILIZATION \

163 --tensor-parallel-size $PREFILLER_TP_SIZE \

162 - --kv-transfer-config '$KV_CONFIG'"

164 + --kv-transfer-config '$KV_CONFIG_P'"

165 if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then

166 IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"

167 for arg in "${extra_args[@]}"; do

@@ -207,7 +209,7 @@ run_tests_for_model() {

209 --enforce-eager \

210 --block-size ${DECODE_BLOCK_SIZE} \

211 --gpu-memory-utilization $GPU_MEMORY_UTILIZATION \

210 - --kv-transfer-config '$KV_CONFIG'"

212 + --kv-transfer-config '$KV_CONFIG_D'"

213 if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then

214 IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"

215 for arg in "${extra_args[@]}"; do

tests/v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh

+6 -4

@@ -193,9 +193,11 @@ run_test_for_device() {

193 local kv_device=$1

194

195 if [[ "$kv_device" == "cuda" ]]; then

196 - local kv_config='{"kv_connector":"NixlConnector","kv_role":"kv_both"}'

196 + local kv_config_p='{"kv_connector":"NixlConnector","kv_role":"kv_producer"}'

197 + local kv_config_d='{"kv_connector":"NixlConnector","kv_role":"kv_consumer"}'

198 else

198 - local kv_config="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"${kv_device}\"}"

199 + local kv_config_p="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_producer\",\"kv_buffer_device\":\"${kv_device}\"}"

200 + local kv_config_d="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_consumer\",\"kv_buffer_device\":\"${kv_device}\"}"

201 fi

202

203 echo ""

@@ -248,7 +250,7 @@ run_test_for_device() {

250 --block-size ${BLOCK_SIZE} \

251 --gpu-memory-utilization $GPU_MEMORY_UTILIZATION \

252 --tensor-parallel-size $PREFILLER_TP_SIZE \

251 - --kv-transfer-config "$kv_config" \

253 + --kv-transfer-config "$kv_config_p" \

254 --speculative-config "$PREFILL_SPEC_CONFIG" \

255 --attention-backend $ATTENTION_BACKEND \

256 ${EXTRA_SERVE_ARGS[@]+"${EXTRA_SERVE_ARGS[@]}"} &

@@ -287,7 +289,7 @@ run_test_for_device() {

289 --block-size ${BLOCK_SIZE} \

290 --gpu-memory-utilization $GPU_MEMORY_UTILIZATION \

291 --tensor-parallel-size $DECODER_TP_SIZE \

290 - --kv-transfer-config "$kv_config" \

292 + --kv-transfer-config "$kv_config_d" \

293 --speculative-config "$DECODE_SPEC_CONFIG" \

294 --attention-backend $ATTENTION_BACKEND \

295 ${EXTRA_SERVE_ARGS[@]+"${EXTRA_SERVE_ARGS[@]}"} &

tests/v1/kv_connector/unit/test_multi_connector.py

+1 -1

@@ -1058,7 +1058,7 @@ def test_multi_connector_mixed_hma_disables_hybrid_kv_cache(monkeypatch):

1058 "connectors": [

1059 {

1060 "kv_connector": "NixlConnector",

1061 - "kv_role": "kv_both",

1061 + "kv_role": "kv_consumer",

1062 },

1063 {

1064 "kv_connector": "MockConnector",

tests/v1/kv_connector/unit/test_nixl_connector.py

+48 -1

@@ -1363,7 +1363,7 @@ def test_abort_timeout_on_prefiller(monkeypatch, distributed_executor_backend):

1363 timeout = 6

1364 kv_transfer_config = KVTransferConfig(

1365 kv_connector="NixlConnector",

1366 - kv_role="kv_both",

1366 + kv_role="kv_consumer",

1367 kv_connector_extra_config={"kv_lease_duration": timeout},

1368 )

1369 llm_kwargs = {

@@ -2737,3 +2737,50 @@ def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario)

2737 f"got {notif!r} (expected {expected_notif!r}, "

2738 f"buggy form would be {bad_notif!r})"

2739 )

2740 +

2741 +

2742 +def test_kv_both_deprecation_warning(default_vllm_config, dist_init):

2743 + """kv_role='kv_both' should emit a deprecation log warning."""

2744 + from unittest.mock import patch

2745 +

2746 + from vllm.logger import _print_warning_once

2747 +

2748 + _print_warning_once.cache_clear()

2749 +

2750 + vllm_config = create_vllm_config(kv_role="kv_both")

2751 +

2752 + with patch(

2753 + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector.logger"

2754 + ) as mock_logger:

2755 + mock_logger.warning_once = mock_logger.warning_once

2756 + NixlConnector(

2757 + vllm_config,

2758 + KVConnectorRole.WORKER,

2759 + make_kv_cache_config(block_size=16),

2760 + )

2761 +

2762 + mock_logger.warning_once.assert_called_once()

2763 + msg = mock_logger.warning_once.call_args[0][0]

2764 + assert "kv_role='kv_both'" in msg

2765 + assert "deprecated" in msg

2766 +

2767 +

2768 +def test_explicit_kv_role_no_deprecation_warning(default_vllm_config, dist_init):

2769 + """kv_role='kv_consumer' or 'kv_producer' should NOT emit a warning."""

2770 + from unittest.mock import patch

2771 +

2772 + for role in ("kv_consumer", "kv_producer"):

2773 + vllm_config = create_vllm_config(kv_role=role)

2774 + with patch(

2775 + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector.logger"

2776 + ) as mock_logger:

2777 + NixlConnector(

2778 + vllm_config,

2779 + KVConnectorRole.WORKER,

2780 + make_kv_cache_config(block_size=16),

2781 + )

2782 +

2783 + (

2784 + mock_logger.warning_once.assert_not_called(),

2785 + (f"kv_role={role!r} should not emit deprecation warning"),

2786 + )

tests/v1/kv_connector/unit/test_nixl_connector_hma.py

+1 -1

@@ -453,7 +453,7 @@ def test_fewer_blocks_with_hma(monkeypatch, model_name, sw_size):

453 """

454 kv_transfer_config = KVTransferConfig(

455 kv_connector="NixlConnector",

456 - kv_role="kv_both",

456 + kv_role="kv_consumer",

457 )

458 block_size = 16

459 llm_kwargs = {

tests/v1/kv_connector/unit/test_rixl_gpu_mem_diag.py

+1 -1

@@ -75,7 +75,7 @@ def test_gpu_memory_rixl_hma(model_name, sw_size):

75 "gpu_memory_utilization": 0.5,

76 "kv_transfer_config": KVTransferConfig(

77 kv_connector="NixlConnector",

78 - kv_role="kv_both",

78 + kv_role="kv_consumer",

79 ),

80 "max_model_len": 2048,

81 "disable_hybrid_kv_cache_manager": False,

tests/v1/kv_connector/unit/utils.py

+1 -1

@@ -103,7 +103,7 @@ def create_vllm_config(

103 kv_load_failure_policy: Literal["recompute", "fail"] = "fail",

104 kv_connector: str = "NixlConnector",

105 kv_connector_module_path: str | None = None,

106 - kv_role: str = "kv_both",

106 + kv_role: str = "kv_consumer",

107 disable_hybrid_kv_cache_manager: bool | None = None,

108 ) -> VllmConfig:

109 """Initialize VllmConfig For Testing."""

vllm/distributed/kv_transfer/kv_connector/v1/nixl/connector.py

@@ -94,6 +94,15 @@ class NixlConnector(KVConnectorBase_V1, SupportsHMA):

94 super().__init__(vllm_config, role, kv_cache_config)

95 assert vllm_config.kv_transfer_config is not None

96 assert vllm_config.kv_transfer_config.engine_id is not None

97 +

98 + if vllm_config.kv_transfer_config.kv_role == "kv_both":

99 + logger.warning_once(

100 + "Using kv_role='kv_both' with NixlConnector is deprecated "

101 + "and will be removed in a future release. Please set "

102 + "kv_role='kv_producer' for prefill instances and "

103 + "kv_role='kv_consumer' for decode instances. "

104 + )

105 +

106 self.kv_cache_config = kv_cache_config

107 self.engine_id: EngineId = vllm_config.kv_transfer_config.engine_id

108 self.kv_transfer_config = vllm_config.kv_transfer_config