A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 22:45 UTC
[kv_offload] Set offloading connector to prefer HND layout (#41928)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
Martin Hickey committed
May 11, 2026 at 13:05 UTC
8415bf2cdbdd00c1786abb766df568d5ae19e2c0
1 file changed
+4
vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
+4
@@ -163,6 +163,10 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
163
assert self.connector_scheduler is not None
164
return self.connector_scheduler.take_events()
165
166
+ @classmethod
167
+ def get_required_kvcache_layout(cls, vllm_config: VllmConfig) -> str | None:
168
+ return "HND"
169
+
170
def get_kv_connector_stats(self) -> KVConnectorStats | None:
171
if self.connector_worker is None:
172
return None # We only emit stats from the worker-side