A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 19:00 UTC
[kv_offload] Add req_id to ReqContext for per-request tracking (#42507)
Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
Ronen Schaffer committed
May 13, 2026 at 14:11 UTC
79fd1bc7eda1854ba6b243c737ecc4bad74ffb84
4 files changed
+10
-4
tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py
+1
-1
@@ -627,7 +627,7 @@ def _make_scheduler_with_lookup(
627
return scheduler
628
629
630
-_EMPTY_REQ_CTX = ReqContext()
630
+_EMPTY_REQ_CTX = ReqContext(req_id="")
631
632
633
class TestMaximalPrefixLookup:
tests/v1/kv_offload/cpu/test_manager.py
+4
-2
@@ -19,9 +19,11 @@ from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
19
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
20
21
22
-def make_req_context(kv_transfer_params: dict | None = None) -> ReqContext:
22
+def make_req_context(
23
+ req_id: str = "", kv_transfer_params: dict | None = None
24
+) -> ReqContext:
25
"""Create a ReqContext as production code would, from a request's params."""
24
- return ReqContext(kv_transfer_params=kv_transfer_params)
26
+ return ReqContext(req_id=req_id, kv_transfer_params=kv_transfer_params)
27
28
29
_EMPTY_REQ_CTX = make_req_context()
vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
+4
-1
@@ -138,7 +138,10 @@ class RequestOffloadState:
138
self.group_states = tuple(
139
RequestGroupState() for _ in self.config.kv_group_configs
140
)
141
- self.req_context = ReqContext(kv_transfer_params=self.req.kv_transfer_params)
141
+ self.req_context = ReqContext(
142
+ req_id=self.req.request_id,
143
+ kv_transfer_params=self.req.kv_transfer_params,
144
+ )
145
146
def update_offload_keys(self) -> None:
147
for group_config, group_state in zip(
vllm/v1/kv_offload/base.py
+1
@@ -46,6 +46,7 @@ def get_offload_group_idx(key: OffloadKey) -> int:
46
47
@dataclass
48
class ReqContext:
49
+ req_id: str
50
kv_transfer_params: dict[str, Any] | None = None
51
52