| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Unit tests for bi-directional KV cache transfer between P and D nodes. |
| 4 | |
| 5 | Tests cover the new behaviors added by the bi-directional KV transfer PR: |
| 6 | 1. P-node scheduler lifecycle: P pulls KV from D using remote_block_ids, |
| 7 | eliminating redundant prefill computation in multi-turn conversations. |
| 8 | 2. P-node metadata: NixlConnectorMetadata correctly populates recv metadata |
| 9 | when P pulls KV from D (do_remote_decode=True + remote_block_ids). |
| 10 | 3. P-node worker: start_load_kv processes reqs_to_recv for KV pull from D. |
| 11 | 4. D-node request_finished: returns kv_transfer_params with remote_block_ids |
| 12 | and remote_num_tokens so P can pull KV in future turns. |
| 13 | 5. Edge cases: |
| 14 | - No double read after reschedule (_remote_blocks_processed flag) |
| 15 | - remote_num_tokens bounded by block capacity (num_computed_tokens) |
| 16 | - kv_recompute_threshold skips small transfers |
| 17 | - P-node holds blocks for D after finishing |
| 18 | - Cache MISS first turn falls back to local prefill |
| 19 | - Partial remote coverage: P pulls partial, computes the rest |
| 20 | - _remote_blocks_processed flag persists across reschedules |
| 21 | |
| 22 | P-node flags: do_remote_prefill=False (prefill locally), |
| 23 | do_remote_decode=True (don't decode locally, send KV to D). |
| 24 | P pulls KV from D when remote_block_ids is not None and |
| 25 | external tokens > 0. |
| 26 | """ |
| 27 | |
| 28 | import copy |
| 29 | import time |
| 30 | from unittest.mock import patch |
| 31 | |
| 32 | import pytest |
| 33 | |
| 34 | from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorRole |
| 35 | from vllm.distributed.kv_transfer.kv_connector.v1.nixl import ( |
| 36 | NixlConnector, |
| 37 | NixlConnectorMetadata, |
| 38 | ) |
| 39 | from vllm.forward_context import ForwardContext |
| 40 | from vllm.v1.outputs import ( |
| 41 | EMPTY_MODEL_RUNNER_OUTPUT, |
| 42 | KVConnectorOutput, |
| 43 | ) |
| 44 | from vllm.v1.request import RequestStatus |
| 45 | |
| 46 | from .test_nixl_connector import FakeNixlConnectorWorker, FakeNixlWrapper |
| 47 | from .utils import ( |
| 48 | assert_scheduler_empty, |
| 49 | create_model_runner_output, |
| 50 | create_request, |
| 51 | create_scheduler, |
| 52 | create_vllm_config, |
| 53 | make_kv_cache_config, |
| 54 | ) |
| 55 | |
| 56 | pytestmark = pytest.mark.cpu_test |
| 57 | |
| 58 | # Common extra config for all bi-directional KV transfer tests. |
| 59 | BIDIR_KV_EXTRA_CONFIG = {"bidirectional_kv_xfer": True, "kv_recompute_threshold": 0} |
| 60 | |
| 61 | |
| 62 | # Helpers |
| 63 | |
| 64 | |
| 65 | def _make_p_node_turn2_request( |
| 66 | request_id, block_size, num_tokens, num_remote_blocks=3, remote_num_tokens=None |
| 67 | ): |
| 68 | """Create a P-node Turn 2 request with remote_block_ids from D.""" |
| 69 | request = create_request( |
| 70 | request_id=request_id, |
| 71 | block_size=block_size, |
| 72 | num_tokens=num_tokens, |
| 73 | do_remote_decode=True, |
| 74 | ) |
| 75 | if remote_num_tokens is None: |
| 76 | remote_num_tokens = num_remote_blocks * block_size |
| 77 | request.kv_transfer_params["remote_block_ids"] = [list(range(num_remote_blocks))] |
| 78 | request.kv_transfer_params["remote_num_tokens"] = remote_num_tokens |
| 79 | request.kv_transfer_params["remote_engine_id"] = "decode-engine" |
| 80 | request.kv_transfer_params["remote_request_id"] = f"decode-{request_id}" |
| 81 | request.kv_transfer_params["remote_host"] = "decode-host" |
| 82 | request.kv_transfer_params["remote_port"] = 5678 |
| 83 | return request |
| 84 | |
| 85 | |
| 86 | def _make_connector_with_fake_worker( |
| 87 | hand_shake_latency=0, cycles_before_done=0, do_handshake=True |
| 88 | ): |
| 89 | """Create a NixlConnector with FakeNixlConnectorWorker.""" |
| 90 | vllm_config = create_vllm_config( |
| 91 | kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, block_size=32 |
| 92 | ) |
| 93 | kv_cache_config = make_kv_cache_config(block_size=32, num_blocks=2) |
| 94 | connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config) |
| 95 | connector.connector_worker = FakeNixlConnectorWorker( |
| 96 | vllm_config, |
| 97 | connector.engine_id, |
| 98 | hand_shake_latency=hand_shake_latency, |
| 99 | kv_cache_config=kv_cache_config, |
| 100 | ) |
| 101 | worker = connector.connector_worker |
| 102 | assert isinstance(worker.nixl_wrapper, FakeNixlWrapper) |
| 103 | worker.kv_cache_layout = "LBHNC" |
| 104 | if do_handshake: |
| 105 | remote_agents, _ = worker._nixl_handshake( |
| 106 | host="localhost", |
| 107 | port=1234, |
| 108 | remote_tp_size=1, |
| 109 | expected_engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID, |
| 110 | ) |
| 111 | worker._remote_agents[FakeNixlConnectorWorker.REMOTE_ENGINE_ID] = remote_agents |
| 112 | return connector, worker |
| 113 | |
| 114 | |
| 115 | def _make_p_node_recv_metadata(request_id, local_blocks, remote_blocks): |
| 116 | """Build NixlConnectorMetadata for P-node pulling KV from D.""" |
| 117 | meta = NixlConnectorMetadata() |
| 118 | meta.add_new_req_to_recv( |
| 119 | request_id=request_id, |
| 120 | local_block_ids=(local_blocks,), |
| 121 | kv_transfer_params={ |
| 122 | "do_remote_prefill": False, |
| 123 | "do_remote_decode": True, |
| 124 | "remote_block_ids": (remote_blocks,), |
| 125 | "remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID, |
| 126 | "remote_request_id": f"decode-{request_id}", |
| 127 | "remote_host": "localhost", |
| 128 | "remote_port": 1234, |
| 129 | "remote_tp_size": 1, |
| 130 | }, |
| 131 | ) |
| 132 | return meta |
| 133 | |
| 134 | |
| 135 | def _do_load_kv(connector, metadata): |
| 136 | """Bind metadata and call start_load_kv.""" |
| 137 | connector.bind_connector_metadata(metadata) |
| 138 | ctx = ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={}) |
| 139 | connector.start_load_kv(ctx) |
| 140 | |
| 141 | |
| 142 | # 1. P-node scheduler lifecycle tests |
| 143 | |
| 144 | |
| 145 | def test_multiturn_lifecycle(): |
| 146 | """Full two-turn lifecycle on the P node: |
| 147 | Turn 1: P prefills locally (do_remote_prefill=False), sends KV to D |
| 148 | (do_remote_decode=True). Finishes LENGTH_CAPPED with remote_block_ids. |
| 149 | Turn 2: P receives remote_block_ids from D. P pulls KV from D because |
| 150 | remote_block_ids is not None and external tokens > 0. Computes only |
| 151 | new tokens, finishes LENGTH_CAPPED.""" |
| 152 | vllm_config = create_vllm_config( |
| 153 | kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, |
| 154 | ) |
| 155 | scheduler = create_scheduler(vllm_config) |
| 156 | BS = vllm_config.cache_config.block_size |
| 157 | |
| 158 | t1 = create_request( |
| 159 | request_id=100, block_size=BS, num_tokens=int(BS * 2.5), do_remote_decode=True |
| 160 | ) |
| 161 | scheduler.add_request(t1) |
| 162 | t1_id = t1.request_id |
| 163 | so = scheduler.schedule() |
| 164 | mro = create_model_runner_output(reqs=[t1]) |
| 165 | eco = scheduler.update_from_output(so, mro) |
| 166 | assert t1.status == RequestStatus.FINISHED_LENGTH_CAPPED |
| 167 | kv = eco[0].outputs[0].kv_transfer_params |
| 168 | assert kv and sum(len(g) for g in kv["remote_block_ids"]) > 0 |
| 169 | so = scheduler.schedule() |
| 170 | scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) |
| 171 | |
| 172 | t2 = _make_p_node_turn2_request(200, BS, int(BS * 2.5)) |
| 173 | scheduler.add_request(t2) |
| 174 | t2_id = t2.request_id |
| 175 | so = scheduler.schedule() |
| 176 | assert t2.status == RequestStatus.WAITING_FOR_REMOTE_KVS |
| 177 | scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) |
| 178 | so = scheduler.schedule() |
| 179 | mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) |
| 180 | mro.kv_connector_output = KVConnectorOutput(finished_recving={t2_id}) |
| 181 | scheduler.update_from_output(so, mro) |
| 182 | so = scheduler.schedule() |
| 183 | mro = create_model_runner_output(reqs=[t2]) |
| 184 | scheduler.update_from_output(so, mro) |
| 185 | assert t2.status == RequestStatus.FINISHED_LENGTH_CAPPED |
| 186 | so = scheduler.schedule() |
| 187 | scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) |
| 188 | so = scheduler.schedule() |
| 189 | mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) |
| 190 | mro.kv_connector_output = KVConnectorOutput(finished_sending={t1_id, t2_id}) |
| 191 | scheduler.update_from_output(so, mro) |
| 192 | assert_scheduler_empty(scheduler) |
| 193 | |
| 194 | |
| 195 | def test_first_turn_no_remote_blocks(): |
| 196 | """First turn: P has no remote_block_ids from D yet. |
| 197 | Standard local prefill, returns kv_transfer_params for future turns.""" |
| 198 | vllm_config = create_vllm_config( |
| 199 | kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, |
| 200 | ) |
| 201 | scheduler = create_scheduler(vllm_config) |
| 202 | BS = vllm_config.cache_config.block_size |
| 203 | req = create_request( |
| 204 | request_id=3, block_size=BS, num_tokens=int(BS * 2.5), do_remote_decode=True |
| 205 | ) |
| 206 | scheduler.add_request(req) |
| 207 | req_id = req.request_id |
| 208 | so = scheduler.schedule() |
| 209 | assert req.status != RequestStatus.WAITING_FOR_REMOTE_KVS |
| 210 | mro = create_model_runner_output(reqs=[req]) |
| 211 | eco = scheduler.update_from_output(so, mro) |
| 212 | assert req.status == RequestStatus.FINISHED_LENGTH_CAPPED |
| 213 | assert eco[0].outputs[0].kv_transfer_params is not None |
| 214 | so = scheduler.schedule() |
| 215 | scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) |
| 216 | so = scheduler.schedule() |
| 217 | mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) |
| 218 | mro.kv_connector_output = KVConnectorOutput(finished_sending={req_id}) |
| 219 | scheduler.update_from_output(so, mro) |
| 220 | assert_scheduler_empty(scheduler) |
| 221 | |
| 222 | |
| 223 | def test_abort_p_side_during_send(): |
| 224 | """P-side do_remote_decode=True: blocks held until finished_sending.""" |
| 225 | vllm_config = create_vllm_config( |
| 226 | kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, |
| 227 | ) |
| 228 | scheduler = create_scheduler(vllm_config) |
| 229 | BS = vllm_config.cache_config.block_size |
| 230 | req = create_request( |
| 231 | request_id=42, block_size=BS, num_tokens=int(BS * 2.5), do_remote_decode=True |
| 232 | ) |
| 233 | scheduler.add_request(req) |
| 234 | req_id = req.request_id |
| 235 | so = scheduler.schedule() |
| 236 | mro = create_model_runner_output(reqs=[req]) |
| 237 | scheduler.update_from_output(so, mro) |
| 238 | assert req_id in scheduler.requests |
| 239 | so = scheduler.schedule() |
| 240 | scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) |
| 241 | assert req_id in scheduler.requests |
| 242 | so = scheduler.schedule() |
| 243 | mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) |
| 244 | mro.kv_connector_output = KVConnectorOutput(finished_sending={req_id}) |
| 245 | scheduler.update_from_output(so, mro) |
| 246 | assert_scheduler_empty(scheduler) |
| 247 | |
| 248 | |
| 249 | def test_abort_p_side_non_length_capped(): |
| 250 | """P-side abort with non-LENGTH_CAPPED → immediate block free.""" |
| 251 | vllm_config = create_vllm_config( |
| 252 | kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, |
| 253 | ) |
| 254 | scheduler = create_scheduler(vllm_config) |
| 255 | BS = vllm_config.cache_config.block_size |
| 256 | req = create_request( |
| 257 | request_id=44, block_size=BS, num_tokens=int(BS * 2.5), do_remote_decode=True |
| 258 | ) |
| 259 | req.sampling_params.max_tokens = 100 |
| 260 | req.max_tokens = 100 |
| 261 | scheduler.add_request(req) |
| 262 | req_id = req.request_id |
| 263 | so = scheduler.schedule() |
| 264 | mro = create_model_runner_output(reqs=[req]) |
| 265 | scheduler.update_from_output(so, mro) |
| 266 | scheduler.finish_requests([req_id], RequestStatus.FINISHED_ABORTED) |
| 267 | conn = scheduler.connector.connector_scheduler |
| 268 | assert req_id in conn._reqs_not_processed |
| 269 | assert req_id not in scheduler.requests |
| 270 | so = scheduler.schedule() |
| 271 | scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) |
| 272 | assert_scheduler_empty(scheduler) |
| 273 | |
| 274 | |
| 275 | def test_remote_blocks_exceed_prompt_tokens(): |
| 276 | """D provides more remote tokens than P's prompt needs. |
| 277 | P caps external tokens to prompt length.""" |
| 278 | vllm_config = create_vllm_config( |
| 279 | kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, |
| 280 | ) |
| 281 | scheduler = create_scheduler(vllm_config) |
| 282 | BS = vllm_config.cache_config.block_size |
| 283 | NUM_TOKENS = int(BS * 2.5) |
| 284 | req = _make_p_node_turn2_request( |
| 285 | 300, BS, NUM_TOKENS, num_remote_blocks=5, remote_num_tokens=5 * BS |
| 286 | ) |
| 287 | scheduler.add_request(req) |
| 288 | req_id = req.request_id |
| 289 | so = scheduler.schedule() |
| 290 | assert req.status == RequestStatus.WAITING_FOR_REMOTE_KVS |
| 291 | assert req.num_computed_tokens == NUM_TOKENS |
| 292 | scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) |
| 293 | so = scheduler.schedule() |
| 294 | mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) |
| 295 | mro.kv_connector_output = KVConnectorOutput(finished_recving={req_id}) |
| 296 | scheduler.update_from_output(so, mro) |
| 297 | so = scheduler.schedule() |
| 298 | mro = create_model_runner_output(reqs=[req]) |
| 299 | scheduler.update_from_output(so, mro) |
| 300 | assert req.status == RequestStatus.FINISHED_LENGTH_CAPPED |
| 301 | so = scheduler.schedule() |
| 302 | scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) |
| 303 | so = scheduler.schedule() |
| 304 | mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) |
| 305 | mro.kv_connector_output = KVConnectorOutput(finished_sending={req_id}) |
| 306 | scheduler.update_from_output(so, mro) |
| 307 | assert_scheduler_empty(scheduler) |
| 308 | |
| 309 | |
| 310 | def test_p_node_pulls_partial_last_block_from_d(): |
| 311 | """D sends remote_block_ids with partially filled last block. |
| 312 | remote_num_tokens < len(remote_block_ids) * block_size. |
| 313 | P pulls only remote_num_tokens worth of external tokens.""" |
| 314 | vllm_config = create_vllm_config( |
| 315 | kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, |
| 316 | ) |
| 317 | scheduler = create_scheduler(vllm_config) |
| 318 | BS = vllm_config.cache_config.block_size |
| 319 | num_remote_blocks = 3 |
| 320 | remote_num_tokens = int(BS * 2.5) |
| 321 | assert remote_num_tokens < num_remote_blocks * BS |
| 322 | NUM_TOKENS = int(BS * 3.5) |
| 323 | req = _make_p_node_turn2_request( |
| 324 | 400, |
| 325 | BS, |
| 326 | NUM_TOKENS, |
| 327 | num_remote_blocks=num_remote_blocks, |
| 328 | remote_num_tokens=remote_num_tokens, |
| 329 | ) |
| 330 | scheduler.add_request(req) |
| 331 | req_id = req.request_id |
| 332 | so = scheduler.schedule() |
| 333 | assert req.status == RequestStatus.WAITING_FOR_REMOTE_KVS |
| 334 | scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) |
| 335 | so = scheduler.schedule() |
| 336 | mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) |
| 337 | mro.kv_connector_output = KVConnectorOutput(finished_recving={req_id}) |
| 338 | scheduler.update_from_output(so, mro) |
| 339 | so = scheduler.schedule() |
| 340 | assert len(scheduler.running) == 1 |
| 341 | mro = create_model_runner_output(reqs=[req]) |
| 342 | scheduler.update_from_output(so, mro) |
| 343 | assert req.status == RequestStatus.FINISHED_LENGTH_CAPPED |
| 344 | so = scheduler.schedule() |
| 345 | scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) |
| 346 | so = scheduler.schedule() |
| 347 | mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) |
| 348 | mro.kv_connector_output = KVConnectorOutput(finished_sending={req_id}) |
| 349 | scheduler.update_from_output(so, mro) |
| 350 | assert_scheduler_empty(scheduler) |
| 351 | |
| 352 | |
| 353 | # 2. P-node metadata tests |
| 354 | |
| 355 | |
| 356 | def test_add_new_req_to_recv_populates_remote_meta(): |
| 357 | """add_new_req_to_recv correctly populates RemoteMeta for P-node |
| 358 | bi-directional KV pull from D.""" |
| 359 | meta = NixlConnectorMetadata() |
| 360 | kv_params = { |
| 361 | "remote_block_ids": [[0, 1, 2]], |
| 362 | "remote_engine_id": "decode-engine", |
| 363 | "remote_request_id": "decode-req-123", |
| 364 | "remote_host": "decode-host", |
| 365 | "remote_port": 5678, |
| 366 | } |
| 367 | local_block_ids = ([10, 11, 12],) |
| 368 | meta.add_new_req_to_recv( |
| 369 | request_id="test-req", |
| 370 | local_block_ids=local_block_ids, |
| 371 | kv_transfer_params=kv_params, |
| 372 | ) |
| 373 | assert "test-req" in meta.reqs_to_recv |
| 374 | rm = meta.reqs_to_recv["test-req"] |
| 375 | assert rm.remote is not None |
| 376 | assert rm.remote.block_ids == kv_params["remote_block_ids"] |
| 377 | assert rm.remote.engine_id == "decode-engine" |
| 378 | assert rm.remote.request_id == "decode-req-123" |
| 379 | assert rm.remote.host == "decode-host" |
| 380 | assert rm.remote.port == 5678 |
| 381 | assert rm.local_block_ids == local_block_ids |
| 382 | |
| 383 | |
| 384 | def test_build_connector_meta_recv_entries(): |
| 385 | """P-node scheduler: do_remote_decode=True + remote_block_ids → |
| 386 | _reqs_need_recv populated, build_connector_meta produces reqs_to_recv.""" |
| 387 | vllm_config = create_vllm_config( |
| 388 | kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, |
| 389 | ) |
| 390 | scheduler = create_scheduler(vllm_config) |
| 391 | BS = vllm_config.cache_config.block_size |
| 392 | req = _make_p_node_turn2_request(1, BS, int(BS * 2.5)) |
| 393 | scheduler.add_request(req) |
| 394 | req_id = req.request_id |
| 395 | so = scheduler.schedule() |
| 396 | assert req.status == RequestStatus.WAITING_FOR_REMOTE_KVS |
| 397 | meta = so.kv_connector_metadata |
| 398 | assert isinstance(meta, NixlConnectorMetadata) |
| 399 | assert req_id in meta.reqs_to_recv |
| 400 | rm = meta.reqs_to_recv[req_id] |
| 401 | assert rm.remote is not None |
| 402 | assert rm.remote.engine_id == "decode-engine" |
| 403 | |
| 404 | |
| 405 | def test_build_connector_meta_clears_reqs_need_recv(): |
| 406 | """After build_connector_meta, _reqs_need_recv is cleared.""" |
| 407 | vllm_config = create_vllm_config( |
| 408 | kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, |
| 409 | ) |
| 410 | scheduler = create_scheduler(vllm_config) |
| 411 | BS = vllm_config.cache_config.block_size |
| 412 | req = _make_p_node_turn2_request(2, BS, int(BS * 2.5)) |
| 413 | scheduler.add_request(req) |
| 414 | conn = scheduler.connector.connector_scheduler |
| 415 | scheduler.schedule() |
| 416 | assert len(conn._reqs_need_recv) == 0 |
| 417 | |
| 418 | |
| 419 | def test_build_connector_meta_multiple_requests(): |
| 420 | """Multiple P-node requests all included in reqs_to_recv.""" |
| 421 | vllm_config = create_vllm_config( |
| 422 | kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, |
| 423 | ) |
| 424 | scheduler = create_scheduler(vllm_config) |
| 425 | BS = vllm_config.cache_config.block_size |
| 426 | reqs = [_make_p_node_turn2_request(10 + i, BS, int(BS * 2.5)) for i in range(3)] |
| 427 | for r in reqs: |
| 428 | scheduler.add_request(r) |
| 429 | so = scheduler.schedule() |
| 430 | meta = so.kv_connector_metadata |
| 431 | assert isinstance(meta, NixlConnectorMetadata) |
| 432 | assert len(meta.reqs_to_recv) == 3 |
| 433 | for r in reqs: |
| 434 | assert r.request_id in meta.reqs_to_recv |
| 435 | |
| 436 | |
| 437 | # 3. P-node worker tests (FakeNixlWrapper) |
| 438 | |
| 439 | |
| 440 | @patch( |
| 441 | "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", |
| 442 | FakeNixlWrapper, |
| 443 | ) |
| 444 | def test_p_node_pull_kv_from_d(dist_init): |
| 445 | """P node pulls KV from D via start_load_kv with reqs_to_recv.""" |
| 446 | connector, worker = _make_connector_with_fake_worker() |
| 447 | meta = _make_p_node_recv_metadata("req-p1", [10, 11, 12], [20, 21, 22]) |
| 448 | _do_load_kv(connector, meta) |
| 449 | assert "req-p1" in worker._recving_metadata |
| 450 | _, done_recving = connector.get_finished(finished_req_ids=set()) |
| 451 | assert "req-p1" in done_recving |
| 452 | |
| 453 | |
| 454 | @patch( |
| 455 | "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", |
| 456 | FakeNixlWrapper, |
| 457 | ) |
| 458 | def test_p_node_pull_then_send_kv(dist_init): |
| 459 | """Full P-node bi-directional: pull KV from D → prefill → |
| 460 | send KV back to D via notification.""" |
| 461 | connector, worker = _make_connector_with_fake_worker() |
| 462 | meta = _make_p_node_recv_metadata("req-p2", [10, 11], [20, 21]) |
| 463 | _do_load_kv(connector, meta) |
| 464 | _, done_recving = connector.get_finished(finished_req_ids=set()) |
| 465 | assert "req-p2" in done_recving |
| 466 | worker._reqs_to_send["req-p2"] = time.perf_counter() + 60 |
| 467 | worker._reqs_to_process.add("req-p2") |
| 468 | notif = f"req-p2:{worker.world_size}".encode() |
| 469 | orig = worker.nixl_wrapper.get_new_notifs |
| 470 | worker.nixl_wrapper.get_new_notifs = lambda: {"agent": [notif]} |
| 471 | done_sending, _ = connector.get_finished(finished_req_ids=set()) |
| 472 | assert "req-p2" in done_sending |
| 473 | worker.nixl_wrapper.get_new_notifs = orig |
| 474 | |
| 475 | |
| 476 | @patch( |
| 477 | "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", |
| 478 | FakeNixlWrapper, |
| 479 | ) |
| 480 | def test_p_node_deferred_pull_on_no_handshake(dist_init): |
| 481 | """P defers KV pull when no prior handshake exists.""" |
| 482 | connector, worker = _make_connector_with_fake_worker( |
| 483 | hand_shake_latency=0, do_handshake=False |
| 484 | ) |
| 485 | meta = _make_p_node_recv_metadata("req-p3", [10, 11], [20, 21]) |
| 486 | _do_load_kv(connector, meta) |
| 487 | assert "req-p3" in worker._recving_metadata |
| 488 | timeout = 3.0 |
| 489 | start = time.perf_counter() |
| 490 | while time.perf_counter() - start < timeout: |
| 491 | connector.bind_connector_metadata(NixlConnectorMetadata()) |
| 492 | ctx = ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={}) |
| 493 | connector.start_load_kv(ctx) |
| 494 | _, done = connector.get_finished(finished_req_ids=set()) |
| 495 | if "req-p3" in done: |
| 496 | return |
| 497 | time.sleep(0.2) |
| 498 | raise AssertionError("Transfer did not complete after async handshake") |
| 499 | |
| 500 | |
| 501 | # 4. D-node request_finished returns kv_transfer_params (new behavior) |
| 502 | |
| 503 | |
| 504 | def test_d_node_request_finished_returns_kv_params(): |
| 505 | """D-node request_finished returns kv_transfer_params with |
| 506 | do_remote_decode=True, remote_block_ids, remote_num_tokens |
| 507 | for P to pull. These params go directly to P node.""" |
| 508 | vllm_config = create_vllm_config( |
| 509 | kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, |
| 510 | ) |
| 511 | scheduler = create_scheduler(vllm_config) |
| 512 | BS = vllm_config.cache_config.block_size |
| 513 | req = create_request( |
| 514 | request_id=1, block_size=BS, num_tokens=int(BS * 2.5), do_remote_prefill=True |
| 515 | ) |
| 516 | scheduler.add_request(req) |
| 517 | req_id = req.request_id |
| 518 | so = scheduler.schedule() |
| 519 | scheduler.update_from_output( |
| 520 | so, create_model_runner_output(reqs=[], finished_recving={req_id}) |
| 521 | ) |
| 522 | so = scheduler.schedule() |
| 523 | eco = scheduler.update_from_output( |
| 524 | so, create_model_runner_output(reqs=[req], use_eos=True) |
| 525 | ) |
| 526 | assert req.status == RequestStatus.FINISHED_STOPPED |
| 527 | kv = eco[0].outputs[0].kv_transfer_params |
| 528 | assert kv is not None |
| 529 | assert kv["do_remote_decode"] is True |
| 530 | assert kv["do_remote_prefill"] is False |
| 531 | assert "remote_block_ids" in kv |
| 532 | assert "remote_num_tokens" in kv |
| 533 | assert kv["remote_num_tokens"] > 0 |
| 534 | |
| 535 | |
| 536 | def test_d_node_request_finished_delays_block_free(): |
| 537 | """D-node holds blocks (delay_free=True) until P reads them.""" |
| 538 | vllm_config = create_vllm_config( |
| 539 | kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, |
| 540 | ) |
| 541 | scheduler = create_scheduler(vllm_config) |
| 542 | BS = vllm_config.cache_config.block_size |
| 543 | req = create_request( |
| 544 | request_id=2, block_size=BS, num_tokens=int(BS * 2.5), do_remote_prefill=True |
| 545 | ) |
| 546 | scheduler.add_request(req) |
| 547 | req_id = req.request_id |
| 548 | so = scheduler.schedule() |
| 549 | scheduler.update_from_output( |
| 550 | so, create_model_runner_output(reqs=[], finished_recving={req_id}) |
| 551 | ) |
| 552 | so = scheduler.schedule() |
| 553 | scheduler.update_from_output( |
| 554 | so, create_model_runner_output(reqs=[req], use_eos=True) |
| 555 | ) |
| 556 | assert req_id in scheduler.requests |
| 557 | conn = scheduler.connector.connector_scheduler |
| 558 | assert req_id in conn._reqs_need_send |
| 559 | |
| 560 | |
| 561 | def test_d_node_request_finished_remote_num_tokens(): |
| 562 | """D-node kv_transfer_params includes correct remote_num_tokens.""" |
| 563 | vllm_config = create_vllm_config( |
| 564 | kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, |
| 565 | ) |
| 566 | scheduler = create_scheduler(vllm_config) |
| 567 | BS = vllm_config.cache_config.block_size |
| 568 | req = create_request( |
| 569 | request_id=3, block_size=BS, num_tokens=int(BS * 2.5), do_remote_prefill=True |
| 570 | ) |
| 571 | scheduler.add_request(req) |
| 572 | req_id = req.request_id |
| 573 | so = scheduler.schedule() |
| 574 | scheduler.update_from_output( |
| 575 | so, create_model_runner_output(reqs=[], finished_recving={req_id}) |
| 576 | ) |
| 577 | so = scheduler.schedule() |
| 578 | eco = scheduler.update_from_output( |
| 579 | so, create_model_runner_output(reqs=[req], use_eos=True) |
| 580 | ) |
| 581 | kv = eco[0].outputs[0].kv_transfer_params |
| 582 | assert kv["remote_num_tokens"] > 0 |
| 583 | assert sum(len(g) for g in kv["remote_block_ids"]) > 0 |
| 584 | |
| 585 | |
| 586 | def test_d_node_partial_last_block_remote_num_tokens(): |
| 587 | """D-node: remote_num_tokens < len(remote_block_ids) * block_size |
| 588 | when last block is partially filled.""" |
| 589 | vllm_config = create_vllm_config( |
| 590 | kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, |
| 591 | ) |
| 592 | scheduler = create_scheduler(vllm_config) |
| 593 | BS = vllm_config.cache_config.block_size |
| 594 | req = create_request( |
| 595 | request_id=5, block_size=BS, num_tokens=int(BS * 2.5), do_remote_prefill=True |
| 596 | ) |
| 597 | scheduler.add_request(req) |
| 598 | req_id = req.request_id |
| 599 | so = scheduler.schedule() |
| 600 | scheduler.update_from_output( |
| 601 | so, create_model_runner_output(reqs=[], finished_recving={req_id}) |
| 602 | ) |
| 603 | so = scheduler.schedule() |
| 604 | eco = scheduler.update_from_output( |
| 605 | so, create_model_runner_output(reqs=[req], use_eos=True) |
| 606 | ) |
| 607 | kv = eco[0].outputs[0].kv_transfer_params |
| 608 | total_blocks = sum(len(g) for g in kv["remote_block_ids"]) |
| 609 | assert total_blocks == 3 |
| 610 | assert kv["remote_num_tokens"] < total_blocks * BS |
| 611 | assert kv["remote_num_tokens"] > 0 |
| 612 | |
| 613 | |
| 614 | # 5. Edge case tests |
| 615 | |
| 616 | |
| 617 | def test_no_double_read_blocks_after_reschedule(): |
| 618 | """Edge case 1: update_state_after_alloc called twice for the same |
| 619 | bidirectional request (once on initial schedule, once after |
| 620 | WAITING_FOR_REMOTE_KVS → reschedule). The _remote_blocks_processed |
| 621 | flag must prevent the request from being added to _reqs_need_recv |
| 622 | twice, which would cause P to read D's blocks twice.""" |
| 623 | vllm_config = create_vllm_config( |
| 624 | kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, |
| 625 | ) |
| 626 | scheduler = create_scheduler(vllm_config) |
| 627 | BS = vllm_config.cache_config.block_size |
| 628 | req = _make_p_node_turn2_request(500, BS, int(BS * 2.5)) |
| 629 | scheduler.add_request(req) |
| 630 | req_id = req.request_id |
| 631 | conn = scheduler.connector.connector_scheduler |
| 632 | |
| 633 | # First schedule: request enters WAITING_FOR_REMOTE_KVS, |
| 634 | # _reqs_need_recv populated then cleared by build_connector_meta. |
| 635 | so = scheduler.schedule() |
| 636 | assert req.status == RequestStatus.WAITING_FOR_REMOTE_KVS |
| 637 | meta = so.kv_connector_metadata |
| 638 | assert isinstance(meta, NixlConnectorMetadata) |
| 639 | assert req_id in meta.reqs_to_recv |
| 640 | # _reqs_need_recv should be cleared after build_connector_meta |
| 641 | assert len(conn._reqs_need_recv) == 0 |
| 642 | |
| 643 | # Simulate recv completion |
| 644 | scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) |
| 645 | so = scheduler.schedule() |
| 646 | mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) |
| 647 | mro.kv_connector_output = KVConnectorOutput(finished_recving={req_id}) |
| 648 | scheduler.update_from_output(so, mro) |
| 649 | |
| 650 | # Second schedule after recv: update_state_after_alloc called again. |
| 651 | # The _remote_blocks_processed flag should prevent re-entry. |
| 652 | so = scheduler.schedule() |
| 653 | meta2 = so.kv_connector_metadata |
| 654 | assert isinstance(meta2, NixlConnectorMetadata) |
| 655 | # Must NOT be in reqs_to_recv again |
| 656 | assert req_id not in meta2.reqs_to_recv |
| 657 | |
| 658 | # Clean up |
| 659 | mro = create_model_runner_output(reqs=[req]) |
| 660 | scheduler.update_from_output(so, mro) |
| 661 | assert req.status == RequestStatus.FINISHED_LENGTH_CAPPED |
| 662 | so = scheduler.schedule() |
| 663 | scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) |
| 664 | so = scheduler.schedule() |
| 665 | mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) |
| 666 | mro.kv_connector_output = KVConnectorOutput(finished_sending={req_id}) |
| 667 | scheduler.update_from_output(so, mro) |
| 668 | assert_scheduler_empty(scheduler) |
| 669 | |
| 670 | |
| 671 | def test_remote_num_tokens_bounded_by_blocks(): |
| 672 | """Edge case 2: D-node request_finished must return |
| 673 | remote_num_tokens <= len(remote_block_ids) * block_size. |
| 674 | request.num_tokens includes the last sampled token which has no KV |
| 675 | in the cache, so remote_num_tokens must use num_computed_tokens.""" |
| 676 | vllm_config = create_vllm_config( |
| 677 | kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, |
| 678 | ) |
| 679 | scheduler = create_scheduler(vllm_config) |
| 680 | BS = vllm_config.cache_config.block_size |
| 681 | req = create_request( |
| 682 | request_id=501, |
| 683 | block_size=BS, |
| 684 | num_tokens=int(BS * 2.5), |
| 685 | do_remote_prefill=True, |
| 686 | ) |
| 687 | scheduler.add_request(req) |
| 688 | req_id = req.request_id |
| 689 | so = scheduler.schedule() |
| 690 | scheduler.update_from_output( |
| 691 | so, create_model_runner_output(reqs=[], finished_recving={req_id}) |
| 692 | ) |
| 693 | so = scheduler.schedule() |
| 694 | eco = scheduler.update_from_output( |
| 695 | so, create_model_runner_output(reqs=[req], use_eos=True) |
| 696 | ) |
| 697 | kv = eco[0].outputs[0].kv_transfer_params |
| 698 | assert kv is not None |
| 699 | total_blocks = sum(len(g) for g in kv["remote_block_ids"]) |
| 700 | max_tokens_in_blocks = total_blocks * BS |
| 701 | assert kv["remote_num_tokens"] <= max_tokens_in_blocks, ( |
| 702 | f"remote_num_tokens ({kv['remote_num_tokens']}) exceeds " |
| 703 | f"block capacity ({max_tokens_in_blocks})" |
| 704 | ) |
| 705 | assert kv["remote_num_tokens"] > 0 |
| 706 | |
| 707 | |
| 708 | def test_kv_recompute_threshold_skips_small_transfer(): |
| 709 | """Edge case 3: When remote tokens are below kv_recompute_threshold, |
| 710 | P should skip the remote pull and compute locally instead of |
| 711 | entering WAITING_FOR_REMOTE_KVS.""" |
| 712 | threshold = 256 |
| 713 | vllm_config = create_vllm_config( |
| 714 | kv_connector_extra_config={ |
| 715 | "bidirectional_kv_xfer": True, |
| 716 | "kv_recompute_threshold": threshold, |
| 717 | }, |
| 718 | ) |
| 719 | scheduler = create_scheduler(vllm_config) |
| 720 | BS = vllm_config.cache_config.block_size |
| 721 | |
| 722 | # Create request where remote tokens (48) < threshold (256) |
| 723 | req = _make_p_node_turn2_request( |
| 724 | 502, |
| 725 | BS, |
| 726 | int(BS * 2.5), |
| 727 | num_remote_blocks=3, |
| 728 | remote_num_tokens=3 * BS, |
| 729 | ) |
| 730 | scheduler.add_request(req) |
| 731 | so = scheduler.schedule() |
| 732 | # Should NOT enter WAITING_FOR_REMOTE_KVS — threshold not met |
| 733 | assert req.status != RequestStatus.WAITING_FOR_REMOTE_KVS |
| 734 | assert req.status == RequestStatus.RUNNING |
| 735 | |
| 736 | # Clean up |
| 737 | mro = create_model_runner_output(reqs=[req]) |
| 738 | scheduler.update_from_output(so, mro) |
| 739 | assert req.status == RequestStatus.FINISHED_LENGTH_CAPPED |
| 740 | so = scheduler.schedule() |
| 741 | scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) |
| 742 | so = scheduler.schedule() |
| 743 | mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) |
| 744 | mro.kv_connector_output = KVConnectorOutput(finished_sending={req.request_id}) |
| 745 | scheduler.update_from_output(so, mro) |
| 746 | assert_scheduler_empty(scheduler) |
| 747 | |
| 748 | |
| 749 | def test_p_node_finished_holds_blocks_for_d(): |
| 750 | """Edge case 4: P-node finishes with FINISHED_LENGTH_CAPPED and |
| 751 | do_remote_decode=True. P must hold blocks (delay_free=True) and |
| 752 | return kv_transfer_params with do_remote_prefill=True so D can |
| 753 | read P's blocks.""" |
| 754 | vllm_config = create_vllm_config( |
| 755 | kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, |
| 756 | ) |
| 757 | scheduler = create_scheduler(vllm_config) |
| 758 | BS = vllm_config.cache_config.block_size |
| 759 | req = create_request( |
| 760 | request_id=503, |
| 761 | block_size=BS, |
| 762 | num_tokens=int(BS * 2.5), |
| 763 | do_remote_decode=True, |
| 764 | ) |
| 765 | scheduler.add_request(req) |
| 766 | req_id = req.request_id |
| 767 | so = scheduler.schedule() |
| 768 | mro = create_model_runner_output(reqs=[req]) |
| 769 | eco = scheduler.update_from_output(so, mro) |
| 770 | assert req.status == RequestStatus.FINISHED_LENGTH_CAPPED |
| 771 | kv = eco[0].outputs[0].kv_transfer_params |
| 772 | assert kv is not None |
| 773 | # P-node finished: should tell D to pull (do_remote_prefill=True) |
| 774 | assert kv["do_remote_prefill"] is True |
| 775 | assert kv["do_remote_decode"] is False |
| 776 | assert "remote_block_ids" in kv |
| 777 | assert sum(len(g) for g in kv["remote_block_ids"]) > 0 |
| 778 | # Blocks should be held (request still tracked) |
| 779 | assert req_id in scheduler.requests |
| 780 | |
| 781 | # Clean up: simulate D reading and notifying |
| 782 | so = scheduler.schedule() |
| 783 | scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) |
| 784 | so = scheduler.schedule() |
| 785 | mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) |
| 786 | mro.kv_connector_output = KVConnectorOutput(finished_sending={req_id}) |
| 787 | scheduler.update_from_output(so, mro) |
| 788 | assert_scheduler_empty(scheduler) |
| 789 | |
| 790 | |
| 791 | def test_cache_miss_first_turn_no_remote_pull(): |
| 792 | """Edge case 5: First turn with do_remote_decode=True but no |
| 793 | remote_block_ids (cache MISS). P should prefill locally with |
| 794 | num_external_tokens=0 and not enter WAITING_FOR_REMOTE_KVS.""" |
| 795 | vllm_config = create_vllm_config( |
| 796 | kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, |
| 797 | ) |
| 798 | scheduler = create_scheduler(vllm_config) |
| 799 | BS = vllm_config.cache_config.block_size |
| 800 | req = create_request( |
| 801 | request_id=504, |
| 802 | block_size=BS, |
| 803 | num_tokens=int(BS * 2.5), |
| 804 | do_remote_decode=True, |
| 805 | ) |
| 806 | # No remote_block_ids set — this is a cache MISS |
| 807 | assert req.kv_transfer_params.get("remote_block_ids") is None |
| 808 | scheduler.add_request(req) |
| 809 | so = scheduler.schedule() |
| 810 | # Should NOT wait for remote KVs |
| 811 | assert req.status != RequestStatus.WAITING_FOR_REMOTE_KVS |
| 812 | assert req.status == RequestStatus.RUNNING |
| 813 | |
| 814 | # Clean up |
| 815 | mro = create_model_runner_output(reqs=[req]) |
| 816 | scheduler.update_from_output(so, mro) |
| 817 | so = scheduler.schedule() |
| 818 | scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) |
| 819 | so = scheduler.schedule() |
| 820 | mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) |
| 821 | mro.kv_connector_output = KVConnectorOutput(finished_sending={req.request_id}) |
| 822 | scheduler.update_from_output(so, mro) |
| 823 | assert_scheduler_empty(scheduler) |
| 824 | |
| 825 | |
| 826 | def test_partial_remote_tokens_less_than_prompt(): |
| 827 | """Edge case 6: D's remote_num_tokens covers only part of P's |
| 828 | prompt. P should pull remote_num_tokens worth of external tokens |
| 829 | and compute the rest locally.""" |
| 830 | vllm_config = create_vllm_config( |
| 831 | kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, |
| 832 | ) |
| 833 | scheduler = create_scheduler(vllm_config) |
| 834 | BS = vllm_config.cache_config.block_size |
| 835 | NUM_TOKENS = int(BS * 4.5) # 72 tokens |
| 836 | # D provides only 2 blocks (32 tokens) out of 72 |
| 837 | req = _make_p_node_turn2_request( |
| 838 | 505, |
| 839 | BS, |
| 840 | NUM_TOKENS, |
| 841 | num_remote_blocks=2, |
| 842 | remote_num_tokens=2 * BS, |
| 843 | ) |
| 844 | scheduler.add_request(req) |
| 845 | req_id = req.request_id |
| 846 | so = scheduler.schedule() |
| 847 | assert req.status == RequestStatus.WAITING_FOR_REMOTE_KVS |
| 848 | # num_computed_tokens should reflect the external tokens pulled |
| 849 | # (capped to remote_num_tokens, not full prompt) |
| 850 | assert req.num_computed_tokens < NUM_TOKENS |
| 851 | |
| 852 | # Complete the transfer and finish |
| 853 | scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) |
| 854 | so = scheduler.schedule() |
| 855 | mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) |
| 856 | mro.kv_connector_output = KVConnectorOutput(finished_recving={req_id}) |
| 857 | scheduler.update_from_output(so, mro) |
| 858 | so = scheduler.schedule() |
| 859 | mro = create_model_runner_output(reqs=[req]) |
| 860 | scheduler.update_from_output(so, mro) |
| 861 | assert req.status == RequestStatus.FINISHED_LENGTH_CAPPED |
| 862 | so = scheduler.schedule() |
| 863 | scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) |
| 864 | so = scheduler.schedule() |
| 865 | mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) |
| 866 | mro.kv_connector_output = KVConnectorOutput(finished_sending={req_id}) |
| 867 | scheduler.update_from_output(so, mro) |
| 868 | assert_scheduler_empty(scheduler) |
| 869 | |
| 870 | |
| 871 | def test_remote_blocks_processed_flag_persists(): |
| 872 | """Edge case 7: After recv completes and request is rescheduled, |
| 873 | the _remote_blocks_processed flag in kv_transfer_params prevents |
| 874 | the bidirectional path from re-entering _reqs_need_recv.""" |
| 875 | vllm_config = create_vllm_config( |
| 876 | kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, |
| 877 | ) |
| 878 | scheduler = create_scheduler(vllm_config) |
| 879 | BS = vllm_config.cache_config.block_size |
| 880 | req = _make_p_node_turn2_request(506, BS, int(BS * 2.5)) |
| 881 | scheduler.add_request(req) |
| 882 | req_id = req.request_id |
| 883 | conn = scheduler.connector.connector_scheduler |
| 884 | |
| 885 | # First schedule → WAITING_FOR_REMOTE_KVS |
| 886 | so = scheduler.schedule() |
| 887 | assert req.status == RequestStatus.WAITING_FOR_REMOTE_KVS |
| 888 | scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) |
| 889 | |
| 890 | # Recv completes |
| 891 | so = scheduler.schedule() |
| 892 | mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) |
| 893 | mro.kv_connector_output = KVConnectorOutput(finished_recving={req_id}) |
| 894 | scheduler.update_from_output(so, mro) |
| 895 | |
| 896 | # Verify the flag is set |
| 897 | assert req.kv_transfer_params.get("_remote_blocks_processed") is True |
| 898 | |
| 899 | # Next schedule: update_state_after_alloc is called again. |
| 900 | # _reqs_need_recv must NOT contain this request. |
| 901 | so = scheduler.schedule() |
| 902 | assert req_id not in conn._reqs_need_recv |
| 903 | meta = so.kv_connector_metadata |
| 904 | assert isinstance(meta, NixlConnectorMetadata) |
| 905 | assert req_id not in meta.reqs_to_recv |
| 906 | |
| 907 | # Clean up |
| 908 | mro = create_model_runner_output(reqs=[req]) |
| 909 | scheduler.update_from_output(so, mro) |
| 910 | so = scheduler.schedule() |
| 911 | scheduler.update_from_output(so, EMPTY_MODEL_RUNNER_OUTPUT) |
| 912 | so = scheduler.schedule() |
| 913 | mro = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT) |
| 914 | mro.kv_connector_output = KVConnectorOutput(finished_sending={req_id}) |
| 915 | scheduler.update_from_output(so, mro) |
| 916 | assert_scheduler_empty(scheduler) |
| 917 | |
| 918 | |
| 919 | # 6. Turn-2 deadline expiry check (P declines reading expired D blocks) |
| 920 | |
| 921 | _REMOTE = FakeNixlConnectorWorker.REMOTE_ENGINE_ID |
| 922 | |
| 923 | |
| 924 | @pytest.mark.parametrize( |
| 925 | ("offset", "expiry_delta", "expect_declined"), |
| 926 | [ |
| 927 | # offset known + deadline expired/near-expiry -> decline & recompute. |
| 928 | pytest.param(0.0, -10.0, True, id="expired"), |
| 929 | # offset applied before comparison: +50s in D's clock is expired locally |
| 930 | # when D is 100s ahead. |
| 931 | pytest.param(100.0, 50.0, True, id="offset_makes_expired"), |
| 932 | # within the default 5s safety margin -> treated as expired. |
| 933 | pytest.param(0.0, 2.0, True, id="near_expiry_within_margin"), |
| 934 | # far-future deadline -> read proceeds. |
| 935 | pytest.param(0.0, 1000.0, False, id="valid_far_deadline"), |
| 936 | # no deadline field (older peer / router did not forward) -> skipped. |
| 937 | pytest.param(0.0, None, False, id="missing_expiry_field"), |
| 938 | ], |
| 939 | ) |
| 940 | @patch( |
| 941 | "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", |
| 942 | FakeNixlWrapper, |
| 943 | ) |
| 944 | def test_turn2_deadline_gate(dist_init, offset, expiry_delta, expect_declined): |
| 945 | """P declines on the turn-2 readback if the deadline is known, |
| 946 | near-expiry and a handshake clock offset is known""" |
| 947 | connector, worker = _make_connector_with_fake_worker() |
| 948 | worker._engine_clock_offset[_REMOTE] = offset |
| 949 | expiry_time = None if expiry_delta is None else time.perf_counter() + expiry_delta |
| 950 | meta = NixlConnectorMetadata() |
| 951 | params = { |
| 952 | "do_remote_prefill": False, |
| 953 | "do_remote_decode": True, |
| 954 | "remote_block_ids": ([20, 21],), |
| 955 | "remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID, |
| 956 | "remote_request_id": "decode-req", |
| 957 | "remote_host": "localhost", |
| 958 | "remote_port": 1234, |
| 959 | "remote_tp_size": 1, |
| 960 | } |
| 961 | if expiry_time is not None: |
| 962 | params["remote_blocks_expiry_time"] = expiry_time |
| 963 | meta.add_new_req_to_recv( |
| 964 | request_id="req", |
| 965 | local_block_ids=([10, 11],), |
| 966 | kv_transfer_params=params, |
| 967 | ) |
| 968 | _do_load_kv(connector, meta) |
| 969 | assert worker.xfer_stats.data["num_kv_expired_reqs"] == ( |
| 970 | [1] if expect_declined else [] |
| 971 | ) |
| 972 | # KV expiry must not pollute the transport failure counters. |
| 973 | assert worker.xfer_stats.data["num_failed_transfers"] == [] |
| 974 | assert worker.xfer_stats.data["num_failed_handshakes"] == [] |
| 975 | _, done_recving = connector.get_finished(finished_req_ids=set()) |
| 976 | assert "req" in done_recving |
| 977 | |
| 978 | |
| 979 | @patch( |
| 980 | "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", |
| 981 | FakeNixlWrapper, |
| 982 | ) |
| 983 | def test_turn2_full_prefix_hit_with_expired_deadline_skips_gate(dist_init): |
| 984 | """A full local prefix hit issues no READ. The expiry gate must be skipped |
| 985 | even when D's deadline is expired.""" |
| 986 | connector, worker = _make_connector_with_fake_worker() |
| 987 | worker._engine_clock_offset[_REMOTE] = 0.0 |
| 988 | |
| 989 | meta = NixlConnectorMetadata() |
| 990 | meta.add_new_req_to_recv( |
| 991 | request_id="req", |
| 992 | local_block_ids=(), # full prefix hit -> zero local block groups |
| 993 | kv_transfer_params={ |
| 994 | "do_remote_prefill": False, |
| 995 | "do_remote_decode": True, |
| 996 | "remote_block_ids": ([20, 21],), |
| 997 | "remote_engine_id": _REMOTE, |
| 998 | "remote_request_id": "decode-req", |
| 999 | "remote_host": "localhost", |
| 1000 | "remote_port": 1234, |
| 1001 | "remote_tp_size": 1, |
| 1002 | # Expired on D's clock; with offset 0 it is expired locally too. |
| 1003 | "remote_blocks_expiry_time": time.perf_counter() - 10.0, |
| 1004 | }, |
| 1005 | ) |
| 1006 | # Must not raise, and the gate must not fire for a no-read prefix hit. |
| 1007 | _do_load_kv(connector, meta) |
| 1008 | assert worker.xfer_stats.data["num_kv_expired_reqs"] == [] |
| 1009 | assert worker.xfer_stats.data["num_failed_transfers"] == [] |
| 1010 | |
| 1011 | |
| 1012 | def test_d_node_request_finished_exports_blocks_expiry_time(): |
| 1013 | """D-node (do_remote_prefill request) exports a future float expiry time.""" |
| 1014 | vllm_config = create_vllm_config(kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG) |
| 1015 | scheduler = create_scheduler(vllm_config) |
| 1016 | BS = vllm_config.cache_config.block_size |
| 1017 | req = create_request( |
| 1018 | request_id=600, block_size=BS, num_tokens=int(BS * 2.5), do_remote_prefill=True |
| 1019 | ) |
| 1020 | scheduler.add_request(req) |
| 1021 | req_id = req.request_id |
| 1022 | so = scheduler.schedule() |
| 1023 | scheduler.update_from_output( |
| 1024 | so, create_model_runner_output(reqs=[], finished_recving={req_id}) |
| 1025 | ) |
| 1026 | so = scheduler.schedule() |
| 1027 | eco = scheduler.update_from_output( |
| 1028 | so, create_model_runner_output(reqs=[req], use_eos=True) |
| 1029 | ) |
| 1030 | kv = eco[0].outputs[0].kv_transfer_params |
| 1031 | assert kv["do_remote_decode"] is True |
| 1032 | assert isinstance(kv["remote_blocks_expiry_time"], float) |
| 1033 | assert kv["remote_blocks_expiry_time"] > time.perf_counter() |
| 1034 | |
| 1035 | |
| 1036 | def test_handshake_listener_appends_perf_counter_frame(): |
| 1037 | """The handshake reply carries a 2nd frame with the listener's live |
| 1038 | perf_counter.""" |
| 1039 | import threading |
| 1040 | |
| 1041 | import msgspec |
| 1042 | import zmq |
| 1043 | |
| 1044 | from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import ( |
| 1045 | NixlBaseConnectorScheduler, |
| 1046 | ) |
| 1047 | from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import GET_META_MSG |
| 1048 | from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import zmq_ctx |
| 1049 | from vllm.utils.network_utils import get_open_port, make_zmq_path |
| 1050 | |
| 1051 | encoded_data = {(0, 0): b"payload-rank-0"} |
| 1052 | ready_event = threading.Event() |
| 1053 | stop_event = threading.Event() |
| 1054 | host = "127.0.0.1" |
| 1055 | port = get_open_port() |
| 1056 | listener = threading.Thread( |
| 1057 | target=NixlBaseConnectorScheduler._nixl_handshake_listener, |
| 1058 | args=(encoded_data, ready_event, stop_event, host, port), |
| 1059 | daemon=True, |
| 1060 | ) |
| 1061 | listener.start() |
| 1062 | try: |
| 1063 | assert ready_event.wait(timeout=5) |
| 1064 | path = make_zmq_path("tcp", host, port) |
| 1065 | with zmq_ctx(zmq.REQ, path) as sock: # type: ignore[attr-defined] |
| 1066 | sock.setsockopt(zmq.RCVTIMEO, 5000) # type: ignore[attr-defined] |
| 1067 | t0 = time.perf_counter() |
| 1068 | sock.send(msgspec.msgpack.encode((GET_META_MSG, 0, 0))) |
| 1069 | parts = sock.recv_multipart() |
| 1070 | t1 = time.perf_counter() |
| 1071 | assert len(parts) == 2 |
| 1072 | assert parts[0] == b"payload-rank-0" |
| 1073 | remote_perf = msgspec.msgpack.decode(parts[1]) |
| 1074 | assert isinstance(remote_perf, float) |
| 1075 | # same-clock => offset is near zero. |
| 1076 | offset = remote_perf - (t0 + t1) / 2 |
| 1077 | assert abs(offset) < 1.0 |
| 1078 | finally: |
| 1079 | stop_event.set() |
| 1080 | listener.join(timeout=5) |