[Core] Configure custom encoder cache managers from VllmConfig (#51251)
Signed-off-by: hotTea <958436561@qq.com> Signed-off-by: HMCCMH <chenminghaoscu@163.com> Co-authored-by: HMCCMH <chenminghaoscu@163.com>
hotTea committed
Aug 13, 2026 at 16:08 UTC
7bbbf7c8e5040f7ebd374e8cbc657e01af1136dd
4 files changed
+36
-8
vllm/config/ec_manager_config.py
+12
-2
@@ -2,6 +2,8 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
from abc import ABC
5
+from dataclasses import field
6
+from typing import Any
7
8
from vllm.config.utils import config
9
from vllm.utils.import_utils import resolve_obj_by_qualname
@@ -10,8 +12,16 @@ from vllm.utils.import_utils import resolve_obj_by_qualname
12
@config
13
class EncoderCacheManagerConfig:
14
encoder_cache_manager_cls: str | None = None
13
- """The qualified class name of the custom encoder cache manager.
14
- """
15
+ """Fully qualified class name of the custom encoder cache manager."""
16
+
17
+ manager_config: dict[str, Any] = field(default_factory=dict)
18
+ """Opaque configuration interpreted by the custom cache manager."""
19
+
20
+ def __post_init__(self) -> None:
21
+ if self.encoder_cache_manager_cls is None and self.manager_config:
22
+ raise ValueError(
23
+ "manager_config requires encoder_cache_manager_cls to be set."
24
+ )
25
26
def get_encoder_cache_manager_obj(self):
27
cls_path = self.encoder_cache_manager_cls
vllm/engine/arg_utils.py
+10
@@ -41,6 +41,7 @@ from vllm.config import (
41
DeviceConfig,
42
DiffusionConfig,
43
ECTransferConfig,
44
+ EncoderCacheManagerConfig,
45
EPLBConfig,
46
FaultToleranceConfig,
47
KernelConfig,
@@ -686,6 +687,9 @@ class EngineArgs:
687
kv_events_config: KVEventsConfig | None = None
688
689
ec_transfer_config: ECTransferConfig | None = None
690
+ ec_manager_config: EncoderCacheManagerConfig = get_field(
691
+ VllmConfig, "ec_manager_config"
692
+ )
693
reasoning_config: ReasoningConfig = get_field(VllmConfig, "reasoning_config")
694
695
generation_config: str = ModelConfig.generation_config
@@ -766,6 +770,8 @@ class EngineArgs:
770
self.mamba_config = MambaConfig(**self.mamba_config)
771
if isinstance(self.kernel_config, dict):
772
self.kernel_config = KernelConfig(**self.kernel_config)
773
+ if isinstance(self.ec_manager_config, dict):
774
+ self.ec_manager_config = EncoderCacheManagerConfig(**self.ec_manager_config)
775
if isinstance(self.eplb_config, dict):
776
self.eplb_config = EPLBConfig(**self.eplb_config)
777
if isinstance(self.weight_transfer_config, dict):
@@ -1625,6 +1631,9 @@ class EngineArgs:
1631
vllm_group.add_argument(
1632
"--ec-transfer-config", **vllm_kwargs["ec_transfer_config"]
1633
)
1634
+ vllm_group.add_argument(
1635
+ "--ec-manager-config", **vllm_kwargs["ec_manager_config"]
1636
+ )
1637
vllm_group.add_argument(
1638
"--compilation-config", "-cc", **vllm_kwargs["compilation_config"]
1639
)
@@ -2506,6 +2515,7 @@ class EngineArgs:
2515
kv_transfer_config=self.kv_transfer_config,
2516
kv_events_config=self.kv_events_config,
2517
ec_transfer_config=self.ec_transfer_config,
2518
+ ec_manager_config=self.ec_manager_config,
2519
reasoning_config=self.reasoning_config,
2520
profiler_config=self.profiler_config,
2521
additional_config=self.additional_config,
vllm/v1/core/encoder_cache_manager.py
+7
@@ -5,6 +5,7 @@ from collections import OrderedDict
5
from collections.abc import Mapping
6
from typing import TYPE_CHECKING
7
8
+from vllm.config import VllmConfig
9
from vllm.config.ec_manager_config import EncoderCacheManagerMetadata
10
from vllm.logger import init_logger
11
from vllm.v1.request import Request
@@ -65,6 +66,12 @@ class EncoderCacheManager:
66
last call to get_freed_mm_hashes(). This list is cleared on return.
67
"""
68
69
+ @classmethod
70
+ def create_manager(
71
+ cls, *, cache_size: int, vllm_config: "VllmConfig"
72
+ ) -> "EncoderCacheManager":
73
+ return cls(cache_size=cache_size)
74
+
75
def __init__(self, cache_size: int):
76
self.cache_size = cache_size
77
self.num_free_slots = cache_size
vllm/v1/core/sched/scheduler.py
+7
-6
@@ -234,14 +234,15 @@ class Scheduler(SchedulerInterface):
234
)
235
encoder_cache_size = mm_budget.encoder_cache_size if mm_budget else 0
236
manager_cls_obj = vllm_config.ec_manager_config.get_encoder_cache_manager_obj()
237
- if manager_cls_obj is not None:
238
- self.encoder_cache_manager = manager_cls_obj(cache_size=encoder_cache_size)
239
- else:
240
- self.encoder_cache_manager = (
241
- EncoderDecoderCacheManager(cache_size=encoder_cache_size)
237
+ if manager_cls_obj is None:
238
+ manager_cls_obj = (
239
+ EncoderDecoderCacheManager
240
if self.is_encoder_decoder
243
- else EncoderCacheManager(cache_size=encoder_cache_size)
241
+ else EncoderCacheManager
242
)
243
+ self.encoder_cache_manager = manager_cls_obj.create_manager(
244
+ cache_size=encoder_cache_size, vllm_config=vllm_config
245
+ )
246
speculative_config = vllm_config.speculative_config
247
self.use_eagle = False
248
self.num_spec_tokens = vllm_config.num_speculative_tokens