@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 758df5afe7

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 10, 00:15 UTC

[NIXL][Metrics] Track `nixl_num_kv_expired_reqs` metric in Prometheus (#32340)

Add a new metric to track the number of requests that had their KV blocks expire. The scenario is particularly important to surface and track as it is a vital indicator of the health of the deployment. Currently we're resorting to track these failures through unstructured log parsing (which is, among other thing, error string dependent); current main: > Releasing expired KV blocks for request cmpl-071d which were retrieved by 0 decode worker(s) within 0 seconds. Signed-off-by: NickLucche <nlucches@redhat.com>

Nicolò Lucchesi committed Jan 19, 2026 at 13:28 UTC 758df5afe7ac8cdc8141fd25f472ee3632377d90

1 file changed +31 -6

vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py

+31 -6

@@ -1930,6 +1930,7 @@ class NixlConnectorWorker:

1930 if now < expires:

1931 break

1932 count = self.consumer_notification_counts_by_req.pop(req_id, 0)

1933 + self.xfer_stats.record_kv_expired_req()

1934 logger.warning(

1935 "Releasing expired KV blocks for request %s which were "

1936 "retrieved by %d decode worker(s) within %d seconds.",

@@ -2499,13 +2500,14 @@ class NixlKVConnectorStats(KVConnectorStats):

2500

2501 def reset(self):

2502 # Must be serializable

2502 - self.data: dict[str, list[float]] = {

2503 + self.data: dict[str, list[float | int]] = {

2504 "transfer_duration": [],

2505 "post_duration": [],

2506 "bytes_transferred": [],

2507 "num_descriptors": [],

2508 "num_failed_transfers": [],

2509 "num_failed_notifications": [],

2510 + "num_kv_expired_reqs": [],

2511 }

2512

2513 def record_transfer(self, res: nixlXferTelemetry):

@@ -2517,11 +2519,15 @@ class NixlKVConnectorStats(KVConnectorStats):

2519

2520 def record_failed_transfer(self):

2521 """Record a failed NIXL transfer operation."""

2520 - self.data["num_failed_transfers"].append(1.0)

2522 + self.data["num_failed_transfers"].append(1)

2523

2524 def record_failed_notification(self):

2525 """Record a failed NIXL notification (send_notif)."""

2524 - self.data["num_failed_notifications"].append(1.0)

2526 + self.data["num_failed_notifications"].append(1)

2527 +

2528 + def record_kv_expired_req(self):

2529 + """Record a request that had its KV blocks expire."""

2530 + self.data["num_kv_expired_reqs"].append(1)

2531

2532 def clone_and_reset(self) -> "NixlKVConnectorStats":

2533 old = copy.copy(self)

@@ -2529,7 +2535,13 @@ class NixlKVConnectorStats(KVConnectorStats):

2535 return old

2536

2537 def is_empty(self) -> bool:

2532 - return self.num_successful_transfers == 0

2538 + # Do not discard metrics update that are entirely failures related.

2539 + return (

2540 + self.num_successful_transfers == 0

2541 + and len(self.data["num_failed_transfers"]) == 0

2542 + and len(self.data["num_failed_notifications"]) == 0

2543 + and len(self.data["num_kv_expired_reqs"]) == 0

2544 + )

2545

2546 def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:

2547 if not other.is_empty():

@@ -2541,7 +2553,9 @@ class NixlKVConnectorStats(KVConnectorStats):

2553

2554 def reduce(self) -> dict[str, int | float]:

2555 # Compute compact representative stats suitable for CLI logging

2544 - if self.is_empty():

2556 + if self.num_successful_transfers == 0:

2557 + # CLI logging only reports successful transfers stats. If all requests in

2558 + # the interval were unsuccessful, Prom will report failures stats instead.

2559 return {

2560 "Num successful transfers": 0,

2561 "Avg xfer time (ms)": 0,

@@ -2677,6 +2691,16 @@ class NixlPromMetrics(KVConnectorPromMetrics):

2691 counter_nixl_num_failed_notifications

2692 )

2693

2694 + counter_nixl_num_kv_expired_reqs = self._counter_cls(

2695 + name="vllm:nixl_num_kv_expired_reqs",

2696 + documentation="Number of requests that had their KV expire. "

2697 + "NOTE: This metric is tracked on the P instance.",

2698 + labelnames=labelnames,

2699 + )

2700 + self.counter_nixl_num_kv_expired_reqs = self.make_per_engine(

2701 + counter_nixl_num_kv_expired_reqs

2702 + )

2703 +

2704 def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):

2705 for prom_obj, list_item_key in zip(

2706 [

@@ -2698,8 +2722,9 @@ class NixlPromMetrics(KVConnectorPromMetrics):

2722 [

2723 self.counter_nixl_num_failed_transfers,

2724 self.counter_nixl_num_failed_notifications,

2725 + self.counter_nixl_num_kv_expired_reqs,

2726 ],

2702 - ["num_failed_transfers", "num_failed_notifications"],

2727 + ["num_failed_transfers", "num_failed_notifications", "num_kv_expired_reqs"],

2728 ):

2729 for list_item in transfer_stats_data[counter_item_key]:

2730 counter_obj[engine_idx].inc(list_item)