A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 12:30 UTC
[Misc] Introduce ec_both role EC (encoder cache) connector (#34182)
Signed-off-by: Qi Wang <qiwa@nvidia.com>
Qi Wang committed
Feb 10, 2026 at 10:55 UTC
33bcd3dc3bf4d581c051400c8d9bb9433d2c87af
3 files changed
+10
-4
vllm/config/ec_transfer.py
+3
-3
@@ -7,8 +7,8 @@ from typing import Any, Literal, get_args
7
8
from vllm.config.utils import config
9
10
-ECProducer = Literal["ec_producer"]
11
-ECConsumer = Literal["ec_consumer"]
10
+ECProducer = Literal["ec_producer", "ec_both"]
11
+ECConsumer = Literal["ec_consumer", "ec_both"]
12
ECRole = Literal[ECProducer, ECConsumer]
13
14
@@ -33,7 +33,7 @@ class ECTransferConfig:
33
34
ec_role: ECRole | None = None
35
"""Whether this vLLM instance produces, consumes EC cache, or both. Choices
36
- are 'ec_producer', 'ec_consumer'."""
36
+ are 'ec_producer', 'ec_consumer', 'ec_both'."""
37
38
ec_rank: int | None = None
39
"""The rank of this vLLM instance in the EC cache transfer. Typical value:
vllm/distributed/ec_transfer/ec_connector/base.py
+5
@@ -63,6 +63,7 @@ class ECConnectorBase(ABC):
63
self._role = role
64
if vllm_config.ec_transfer_config is not None:
65
self._is_producer = vllm_config.ec_transfer_config.is_ec_producer
66
+ self._is_consumer = vllm_config.ec_transfer_config.is_ec_consumer
67
else:
68
raise ValueError("ec_transfer_config must be set for ECConnectorBase")
69
@@ -74,6 +75,10 @@ class ECConnectorBase(ABC):
75
def is_producer(self) -> bool:
76
return self._is_producer
77
78
+ @property
79
+ def is_consumer(self) -> bool:
80
+ return self._is_consumer
81
+
82
# ==============================
83
# Worker-side methods
84
# ==============================
vllm/v1/worker/ec_connector_model_runner_mixin.py
+2
-1
@@ -72,7 +72,8 @@ class ECConnectorModelRunnerMixin:
72
assert scheduler_output.ec_connector_metadata is not None
73
ec_connector.bind_connector_metadata(scheduler_output.ec_connector_metadata)
74
75
- if not ec_connector.is_producer:
75
+ # Load caches for consumer or both roles
76
+ if ec_connector.is_consumer:
77
ec_connector.start_load_caches(encoder_cache, **kwargs)
78
79
try: