@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 164dca8d1e

[Nixl] Separate transport-failure metrics from KV expiry (#55854)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>

Nicolò Lucchesi committed Sep 18, 2026 at 18:08 UTC 164dca8d1e65b8f10346321f01b25e23caa642a6

9 files changed +224 -22

docs/features/nixl_connector_usage.md

+9 -6

@@ -418,12 +418,13 @@ activity for the last reporting interval. Example output:

418 ```text

419 KV Transfer metrics: Num successful transfers=4, Avg xfer time (ms)=1.381,

420 P90 xfer time (ms)=2.601, Avg post time (ms)=0.672, P90 post time (ms)=0.801,

421 -Avg MB per transfer=2.25, Throughput (MB/s)=1629.549, Avg number of descriptors=72.0

421 +Avg MB per transfer=2.25, Throughput (MB/s)=1629.549, Avg number of descriptors=72.0,

422 +Num failed transfers=0, Num KV expired reqs=0

423 ```

424

425 The table below describes each field. All timing values cover only the

425 -successful transfers recorded in the current interval; failed transfers are

426 -counted separately via Prometheus (see

426 +successful transfers recorded in the current interval; failures are reported

427 +as interval counts in the same line and via Prometheus (see

428 [Prometheus metrics](#prometheus-metrics) below).

429

430 | Metric | Unit | Description |

@@ -436,6 +437,8 @@ counted separately via Prometheus (see

437 | `Avg MB per transfer` | MB | Mean payload size per transfer, computed as `total bytes transferred / number of transfers`. Reflects the average KV cache footprint of a single request (sequence length × layers × head dimension × dtype bytes). |

438 | `Throughput (MB/s)` | MB/s | Effective bandwidth over the interval: `total MB transferred / total xfer time (s)` across all successful transfers. This is aggregate throughput, not per-request bandwidth. |

439 | `Avg number of descriptors` | count | Mean number of NIXL memory descriptors (scatter-gather segments) submitted per transfer. More descriptors indicate more fragmented or larger KV cache allocations; very high counts can increase descriptor-registration overhead. |

440 +| `Num failed transfers` | count | Failed NIXL transfers, handshakes, and completion notifications (`send_notif`) in the interval, grouped because all are sporadic lower-transport-layer events. |

441 +| `Num KV expired reqs` | count | Requests whose KV blocks expired before being read. Reported separately from the failure counts above because it is an autoscaler/lease-tuning signal rather than a transport issue. |

442

443 ### Prometheus metrics

444

@@ -448,9 +451,9 @@ exported when NixlConnector is active:

451 | `vllm:nixl_post_time_seconds` | Histogram | Time to submit the transfer request to the RDMA backend (seconds). |

452 | `vllm:nixl_bytes_transferred` | Histogram | Bytes moved per transfer. |

453 | `vllm:nixl_num_descriptors` | Histogram | Descriptor count per transfer. |

451 -| `vllm:nixl_num_failed_transfers` | Counter | Cumulative count of failed NIXL KV-block transfers. |

452 -| `vllm:nixl_num_failed_notifications` | Counter | Cumulative count of failed completion notifications (`send_notif`). |

453 -| `vllm:nixl_num_kv_expired_reqs` | Counter | Requests whose KV blocks expired on the prefiller before the decoder read them (tracked on the P instance). |

454 +| `vllm:nixl_num_failed_transfers` | Counter | Cumulative count of failed NIXL KV-block transfers, including handshake and notification (`send_notif`) failures. These are grouped because all are sporadic lower-transport-layer events. |

455 +| `vllm:nixl_num_failed_notifications` | Counter | Cumulative count of failed notifications (`send_notif`), retained for compatibility. These failures are also included in `vllm:nixl_num_failed_transfers`; do not sum the two counters. |

456 +| `vllm:nixl_num_kv_expired_reqs` | Counter | Requests whose KV blocks expired on the prefiller before the decoder read them (tracked on the P instance). Kept separate from the failure counters above: KV expiry is an autoscaler/lease-tuning signal, not a transport issue. |

457

458 !!! tip

459 High `vllm:nixl_num_kv_expired_reqs` indicates that the prefiller's lease

tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py

@@ -969,6 +969,9 @@ def test_turn2_deadline_gate(dist_init, offset, expiry_delta, expect_declined):

969 assert worker.xfer_stats.data["num_kv_expired_reqs"] == (

970 [1] if expect_declined else []

971 )

972 + # KV expiry must not pollute the transport failure counters.

973 + assert worker.xfer_stats.data["num_failed_transfers"] == []

974 + assert worker.xfer_stats.data["num_failed_handshakes"] == []

975 _, done_recving = connector.get_finished(finished_req_ids=set())

976 assert "req" in done_recving

977

tests/v1/kv_connector/unit/test_multi_connector.py

+24

@@ -567,6 +567,8 @@ class TestMultiConnectorStats:

567 "num_descriptors": [10, 20],

568 "num_failed_transfers": [],

569 "num_failed_notifications": [],

570 + "num_failed_handshakes": [],

571 + "num_kv_expired_reqs": [],

572 }

573 }

574

@@ -590,6 +592,8 @@ class TestMultiConnectorStats:

592 "num_descriptors": [10],

593 "num_failed_transfers": [],

594 "num_failed_notifications": [],

595 + "num_failed_handshakes": [],

596 + "num_kv_expired_reqs": [],

597 },

598 "MockConnector": {"mock_field": [1, 2, 3]},

599 }

@@ -619,6 +623,8 @@ class TestMultiConnectorStats:

623 "num_descriptors": [10],

624 "num_failed_transfers": [],

625 "num_failed_notifications": [],

626 + "num_failed_handshakes": [],

627 + "num_kv_expired_reqs": [],

628 },

629 }

630

@@ -638,6 +644,8 @@ class TestMultiConnectorStats:

644 "num_descriptors": [10],

645 "num_failed_transfers": [],

646 "num_failed_notifications": [],

647 + "num_failed_handshakes": [],

648 + "num_kv_expired_reqs": [],

649 }

650 )

651 mock_stats = MockConnectorStats(data={"mock_field": [1, 2, 3]})

@@ -667,6 +675,8 @@ class TestMultiConnectorStats:

675 "num_descriptors": [10],

676 "num_failed_transfers": [],

677 "num_failed_notifications": [],

678 + "num_failed_handshakes": [],

679 + "num_kv_expired_reqs": [],

680 }

681 )

682

@@ -698,6 +708,8 @@ class TestMultiConnectorStats:

708 "num_descriptors": [10],

709 "num_failed_transfers": [],

710 "num_failed_notifications": [],

711 + "num_failed_handshakes": [],

712 + "num_kv_expired_reqs": [],

713 },

714 "ExampleConnector": {"some_field": [1, 2, 3]},

715 }

@@ -735,6 +747,8 @@ class TestMultiConnectorStats:

747 "num_descriptors": [10],

748 "num_failed_transfers": [],

749 "num_failed_notifications": [],

750 + "num_failed_handshakes": [],

751 + "num_kv_expired_reqs": [],

752 }

753 )

754 }

@@ -750,6 +764,8 @@ class TestMultiConnectorStats:

764 "num_descriptors": [20],

765 "num_failed_transfers": [],

766 "num_failed_notifications": [],

767 + "num_failed_handshakes": [],

768 + "num_kv_expired_reqs": [],

769 }

770 )

771 }

@@ -781,6 +797,8 @@ class TestMultiConnectorStats:

797 "num_descriptors": [10],

798 "num_failed_transfers": [],

799 "num_failed_notifications": [],

800 + "num_failed_handshakes": [],

801 + "num_kv_expired_reqs": [],

802 }

803 )

804 }

@@ -807,6 +825,8 @@ class TestMultiConnectorStats:

825 "num_descriptors": [10, 20],

826 "num_failed_transfers": [],

827 "num_failed_notifications": [],

828 + "num_failed_handshakes": [],

829 + "num_kv_expired_reqs": [],

830 }

831 )

832 }

@@ -833,6 +853,8 @@ class TestMultiConnectorStats:

853 "num_descriptors": [10, 20],

854 "num_failed_transfers": [],

855 "num_failed_notifications": [],

856 + "num_failed_handshakes": [],

857 + "num_kv_expired_reqs": [],

858 }

859 )

860 }

@@ -862,6 +884,8 @@ class TestMultiConnectorStats:

884 "num_descriptors": [10, 20],

885 "num_failed_transfers": [],

886 "num_failed_notifications": [],

887 + "num_failed_handshakes": [],

888 + "num_kv_expired_reqs": [],

889 }

890 )

891 }

tests/v1/kv_connector/unit/test_nixl_connector.py

+82

@@ -1682,6 +1682,83 @@ def test_kv_connector_stats_aggregation():

1682 assert all(not isinstance(v, np.generic) for v in cli_stats.values())

1683

1684

1685 +def test_kv_connector_stats_failure_grouping():

1686 + """Transfer, handshake and notification failures are reported as one

1687 + transport-failure count, while KV expiry is reported separately: the

1688 + former are sporadic lower-transport-layer events, the latter an

1689 + autoscaler signal."""

1690 + stats = NixlKVConnectorStats()

1691 + assert stats.is_empty()

1692 +

1693 + stats.record_failed_transfer()

1694 + stats.record_failed_handshake()

1695 + stats.record_failed_notification()

1696 + stats.record_kv_expired_req()

1697 + assert not stats.is_empty()

1698 +

1699 + # No successful transfers: latency stats are zero but the failure

1700 + # counts still surface.

1701 + reduced = stats.reduce()

1702 + assert reduced["Num successful transfers"] == 0

1703 + assert reduced["Num failed transfers"] == 3

1704 + assert reduced["Num KV expired reqs"] == 1

1705 +

1706 +

1707 +def test_nixl_prom_metrics_group_handshake_with_transfer_failures():

1708 + """vllm:nixl_num_failed_transfers counts handshake and notification

1709 + failures too, while vllm:nixl_num_kv_expired_reqs stays a separate

1710 + counter."""

1711 + from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram

1712 +

1713 + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (

1714 + NixlPromMetrics,

1715 + )

1716 +

1717 + registry = CollectorRegistry()

1718 +

1719 + class RegistryGauge(Gauge):

1720 + def __init__(self, *args, **kwargs):

1721 + super().__init__(*args, registry=registry, **kwargs)

1722 +

1723 + class RegistryCounter(Counter):

1724 + def __init__(self, *args, **kwargs):

1725 + super().__init__(*args, registry=registry, **kwargs)

1726 +

1727 + class RegistryHistogram(Histogram):

1728 + def __init__(self, *args, **kwargs):

1729 + super().__init__(*args, registry=registry, **kwargs)

1730 +

1731 + vllm_config = create_vllm_config()

1732 + metric_types = {

1733 + Gauge: RegistryGauge,

1734 + Counter: RegistryCounter,

1735 + Histogram: RegistryHistogram,

1736 + }

1737 + prom = NixlPromMetrics(

1738 + vllm_config,

1739 + metric_types,

1740 + labelnames=["engine"],

1741 + per_engine_labelvalues={0: ["engine-0"]},

1742 + )

1743 +

1744 + stats = NixlKVConnectorStats()

1745 + stats.record_failed_transfer()

1746 + stats.record_failed_handshake()

1747 + stats.record_failed_notification()

1748 + stats.record_kv_expired_req()

1749 + prom.observe(stats.data, engine_idx=0)

1750 +

1751 + def counter_value(name: str) -> float:

1752 + for metric in registry.collect():

1753 + for sample in metric.samples:

1754 + if sample.name == name:

1755 + return sample.value

1756 + raise AssertionError(f"metric {name} not found in registry")

1757 +

1758 + assert counter_value("vllm:nixl_num_failed_transfers_total") == 3.0

1759 + assert counter_value("vllm:nixl_num_kv_expired_reqs_total") == 1.0

1760 +

1761 +

1762 def test_multi_kv_connector_stats_aggregation():

1763 """Test MultiKVConnectorStats aggregation across TP ranks using

1764 KVOutputAggregator (used by MultiprocExecutor).

@@ -3029,6 +3106,11 @@ def test_handshake_failure_returns_finished(default_vllm_config, dist_init):

3106 assert request_id in done_recving

3107 assert connector.get_block_ids_with_load_errors() == {1, 2, 3}

3108

3109 + # Handshake failures are recorded as transport failures, separately

3110 + # from KV expiry.

3111 + assert connector.connector_worker.xfer_stats.data["num_failed_handshakes"]

3112 + assert connector.connector_worker.xfer_stats.data["num_failed_transfers"] == []

3113 +

3114

3115 @patch(

3116 "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",

tests/v1/kv_connector/unit/test_nixl_push_connector.py

@@ -47,6 +47,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (

47 from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (

48 NixlPushConnectorWorker,

49 )

50 +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (

51 + NixlKVConnectorStats,

52 +)

53 from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import TPMapping

54 from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import (

55 get_base_request_id,

@@ -360,6 +363,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):

363 w._recv_failures = set()

364 w._recving_transfers = defaultdict(list)

365 w._is_hma_required = False

366 + w.xfer_stats = NixlKVConnectorStats()

367 w._reqs_to_process = set()

368 w._reqs_to_send = {}

369 w.consumer_notification_counts_by_req = defaultdict(int)

@@ -730,6 +734,10 @@ def test_do_start_push_kv_drops_request_on_handshake_failure():

734 assert xfer_calls == []

735 assert len(failures) == 1

736 assert failures[0]["failure_type"] == "push_handshake_failed"

737 + # The handshake metric is counted once by _ensure_handshake's own

738 + # callback (stubbed out here); the push side must not double-record it.

739 + assert w.xfer_stats.data["num_failed_handshakes"] == []

740 + assert w.xfer_stats.data["num_failed_transfers"] == []

741

742

743 def test_writer_loop_drains_deferred_push_inbox():

vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py

+28 -5

@@ -1306,6 +1306,8 @@ class NixlBaseConnectorWorker:

1306 error=e,

1307 remote_engine_id=eid,

1308 )

1309 + # Count once per handshake, regardless of waiting requests.

1310 + self.xfer_stats.record_failed_handshake()

1311

1312 fut.add_done_callback(done_callback)

1313 return fut

@@ -2811,13 +2813,17 @@ class NixlBaseConnectorWorker:

2813 done_sending.update(self._replicated_pcp_done_sending)

2814 self._replicated_pcp_done_sending.clear()

2815

2814 - # Process receive failures reported by background threads.

2816 + # Process receive failures reported by background threads. Each

2817 + # producer already recorded a specific metric (handshake or

2818 + # notification), so only the failure bookkeeping runs here.

2819 while not self._failed_recv_reqs.empty():

2820 try:

2821 req_id = self._failed_recv_reqs.get_nowait()

2822 except queue.Empty:

2823 break

2820 - self._handle_failed_transfer(req_id, None, self._recv_failures)

2824 + self._handle_failed_transfer(

2825 + req_id, None, self._recv_failures, record_failed_transfer=False

2826 + )

2827

2828 failed_recv_reqs: set[ReqId] = set()

2829 if self._recv_failures:

@@ -3059,10 +3065,27 @@ class NixlBaseConnectorWorker:

3065 return True

3066

3067 def _handle_failed_transfer(

3062 - self, req_id: str, handle: int | None, failed_req_ids: set[str] | None = None

3068 + self,

3069 + req_id: str,

3070 + handle: int | None,

3071 + failed_req_ids: set[str] | None = None,

3072 + record_failed_transfer: bool = True,

3073 ) -> bool:

3064 - """Record a failure and release its handle, returning False to retain it."""

3065 - self.xfer_stats.record_failed_transfer()

3074 + """Record a failure and release its handle, returning False to retain it.

3075 +

3076 + Args:

3077 + req_id: The request ID.

3078 + handle: The transfer handle.

3079 + failed_req_ids: Requests observed as failed; each stays listed

3080 + until it has no outstanding handles.

3081 + record_failed_transfer: Whether to count the failure toward the

3082 + transport-failure metric. Callers that already recorded a more

3083 + specific metric (eg KV expiry, reported separately from

3084 + transport failures) pass False.

3085 +

3086 + """

3087 + if record_failed_transfer:

3088 + self.xfer_stats.record_failed_transfer()

3089 if failed_req_ids is not None:

3090 failed_req_ids.add(req_id)

3091 return handle is None or self._try_release_xfer_handle(req_id, handle)

vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py

+19 -2

@@ -149,7 +149,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):

149 engine_id,

150 )

151 self.xfer_stats.record_kv_expired_req()

152 - self._handle_failed_transfer(req_id, None, self._recv_failures)

152 + # KV expiry is reported separately from transport failures, so only

153 + # the state cleanup side of _handle_failed_transfer runs here.

154 + self._handle_failed_transfer(

155 + req_id, None, self._recv_failures, record_failed_transfer=False

156 + )

157 return

158

159 if any(len(group) > 0 for group in meta.local_block_ids):

@@ -364,7 +368,20 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):

368 remote_agents = self._remote_agents[meta.remote.engine_id]

369 for rank_to_notify, agent in remote_agents.items():

370 if rank_to_notify != (0, read_specs[0].remote_rank):

367 - self.nixl_wrapper.send_notif(agent, notif_msg=notif_id)

371 + try:

372 + self.nixl_wrapper.send_notif(agent, notif_msg=notif_id)

373 + except Exception as e:

374 + self._log_failure(

375 + failure_type="notification_failed",

376 + msg="Remote rank will not update request state. "

377 + "This may indicate network issues.",

378 + req_id=req_id,

379 + error=e,

380 + dst_engine_id=meta.remote.engine_id,

381 + remote_rank=rank_to_notify[1],

382 + remote_agent_name=agent,

383 + )

384 + self.xfer_stats.record_failed_notification()

385

386 def _read_blocks(

387 self,

vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py

@@ -365,6 +365,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):

365 engine_id,

366 req_id,

367 )

368 + self.xfer_stats.record_failed_notification()

369 self._failed_recv_reqs.put(req_id)

370 return

371 for rank, agent_name in agents.items():

@@ -377,6 +378,9 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):

378 error=e,

379 remote_rank=rank,

380 )

381 + self.xfer_stats.record_failed_notification()

382 + # Earlier registrations may still trigger WRITEs into D's blocks.

383 + # Keep the receive pending until those writes are finished.

384 logger.debug(

385 "Sent PUSH_REG for %s to engine %s (%dB)", req_id, engine_id, len(notif_msg)

386 )

vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py

+47 -9

@@ -39,6 +39,7 @@ class NixlKVConnectorStats(KVConnectorStats):

39 "num_descriptors": [],

40 "num_failed_transfers": [],

41 "num_failed_notifications": [],

42 + "num_failed_handshakes": [],

43 "num_kv_expired_reqs": [],

44 }

@@ -57,6 +58,10 @@ class NixlKVConnectorStats(KVConnectorStats):

58 """Record a failed NIXL notification (send_notif)."""

59 self.data["num_failed_notifications"].append(1)

61 + def record_failed_handshake(self):

62 + """Record a failed NIXL handshake."""

63 + self.data["num_failed_handshakes"].append(1)

64 +

65 def record_kv_expired_req(self):

66 """Record a request that had its KV blocks expire."""

67 self.data["num_kv_expired_reqs"].append(1)

@@ -72,6 +77,7 @@ class NixlKVConnectorStats(KVConnectorStats):

77 self.num_successful_transfers == 0

78 and len(self.data["num_failed_transfers"]) == 0

79 and len(self.data["num_failed_notifications"]) == 0

80 + and len(self.data["num_failed_handshakes"]) == 0

81 and len(self.data["num_kv_expired_reqs"]) == 0

82 )

@@ -84,10 +90,22 @@ class NixlKVConnectorStats(KVConnectorStats):

90 return self

92 def reduce(self) -> dict[str, int | float]:

87 - # Compute compact representative stats suitable for CLI logging

93 + # Compute compact representative stats suitable for CLI logging.

94 + # Failure counts are reported on every interval: transfer, handshake

95 + # and notification failures are grouped as sporadic

96 + # lower-transport-layer events, while KV expiry is reported separately

97 + # as it is an actionable autoscaler signal rather than a transport

98 + # issue.

99 + failure_counts = {

100 + "Num failed transfers": len(self.data["num_failed_transfers"])

101 + + len(self.data["num_failed_handshakes"])

102 + + len(self.data["num_failed_notifications"]),

103 + "Num KV expired reqs": len(self.data["num_kv_expired_reqs"]),

104 + }

105 if self.num_successful_transfers == 0:

89 - # CLI logging only reports successful transfers stats. If all requests in

90 - # the interval were unsuccessful, Prom will report failures stats instead.

106 + # Timing / throughput stats only cover successful transfers. If

107 + # all requests in the interval were unsuccessful, the failure

108 + # counts above still surface what happened.

109 return {

110 "Num successful transfers": 0,

111 "Avg xfer time (ms)": 0,

@@ -97,6 +115,7 @@ class NixlKVConnectorStats(KVConnectorStats):

115 "Avg MB per transfer": 0,

116 "Throughput (MB/s)": 0,

117 "Avg number of descriptors": 0,

118 + **failure_counts,

119 }

120

121 xfer_time = np.asarray(self.data["transfer_duration"])

@@ -122,6 +141,7 @@ class NixlKVConnectorStats(KVConnectorStats):

141 "Avg MB per transfer": round(avg_mb.item(), 3),

142 "Throughput (MB/s)": round(throughput_mb_s.item(), 3),

143 "Avg number of descriptors": round(descs.mean().item(), 1),

144 + **failure_counts,

145 }

146

147 @property

@@ -212,15 +232,20 @@ class NixlPromMetrics(KVConnectorPromMetrics):

232 )

233 counter_nixl_num_failed_transfers = self._counter_cls(

234 name="vllm:nixl_num_failed_transfers",

215 - documentation="Number of failed NIXL KV Cache transfers.",

235 + documentation="Number of failed NIXL KV Cache transfers, including"

236 + " handshake and notification failures. NOTE: KV expiry is tracked"

237 + " separately in vllm:nixl_num_kv_expired_reqs.",

238 labelnames=labelnames,

239 )

240 self.counter_nixl_num_failed_transfers = create_metric_per_engine(

241 counter_nixl_num_failed_transfers, self.per_engine_labelvalues

242 )

243 +

244 counter_nixl_num_failed_notifications = self._counter_cls(

245 name="vllm:nixl_num_failed_notifications",

223 - documentation="Number of failed NIXL KV Cache notifications.",

246 + documentation="Number of failed NIXL KV Cache notifications. "

247 + "Retained for compatibility; these failures are also included in "

248 + "vllm:nixl_num_failed_transfers.",

249 labelnames=labelnames,

250 )

251 self.counter_nixl_num_failed_notifications = create_metric_per_engine(

@@ -254,13 +279,26 @@ class NixlPromMetrics(KVConnectorPromMetrics):

279 ):

280 for list_item in transfer_stats_data[list_item_key]:

281 prom_obj[engine_idx].observe(list_item)

257 - for counter_obj, counter_item_key in zip(

282 + for counter_obj, counter_item_keys in zip(

283 [

284 self.counter_nixl_num_failed_transfers,

285 self.counter_nixl_num_failed_notifications,

286 self.counter_nixl_num_kv_expired_reqs,

287 ],

263 - ["num_failed_transfers", "num_failed_notifications", "num_kv_expired_reqs"],

288 + [

289 + # Transfer, handshake and notification failures are grouped:

290 + # all are sporadic lower-transport-layer events. KV expiry is

291 + # reported separately as it signals autoscaler behavior, not

292 + # transport health.

293 + (

294 + "num_failed_transfers",

295 + "num_failed_handshakes",

296 + "num_failed_notifications",

297 + ),

298 + ("num_failed_notifications",),

299 + ("num_kv_expired_reqs",),

300 + ],

301 ):

265 - for list_item in transfer_stats_data[counter_item_key]:

266 - counter_obj[engine_idx].inc(list_item)

302 + for counter_item_key in counter_item_keys:

303 + for list_item in transfer_stats_data[counter_item_key]:

304 + counter_obj[engine_idx].inc(list_item)