A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 04:15 UTC
[KV Connector] Update lmcache connector with latest compatibility (#27681)
Signed-off-by: Samuel Shen <slshen@uchicago.edu> Co-authored-by: Samuel Shen <slshen@uchicago.edu>
Shaoting committed
Oct 28, 2025 at 22:38 UTC
a4a4f0f61790895f376afbc35ab41e18a796d52a
1 file changed
+20
-2
vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/vllm_v1_adapter.py
+20
-2
@@ -44,8 +44,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.lmcache_integration.utils impo
44
)
45
from vllm.distributed.parallel_state import get_tensor_model_parallel_rank, get_tp_group
46
from vllm.sampling_params import SamplingParams
47
-from vllm.utils import get_kv_cache_torch_dtype
47
from vllm.utils.math_utils import cdiv
48
+from vllm.utils.torch_utils import get_kv_cache_torch_dtype
49
from vllm.v1.core.sched.output import SchedulerOutput
50
from vllm.version import __version__ as VLLM_VERSION
51
@@ -389,7 +389,7 @@ class ReqMeta:
389
390
391
def need_gpu_interm_buffer(lmcache_config: LMCacheEngineConfig):
392
- return lmcache_config.enable_pd
392
+ return not lmcache_config.enable_pd
393
394
395
def _calculate_mtp_layers(vllm_config, model_config):
@@ -403,6 +403,20 @@ def _calculate_mtp_layers(vllm_config, model_config):
403
num_mtp_layers = getattr(
404
model_config.hf_config, "num_nextn_predict_layers", 0
405
)
406
+
407
+ elif vllm_config.speculative_config.use_eagle():
408
+ try:
409
+ draft_model_config = vllm_config.speculative_config.draft_model_config
410
+ num_mtp_layers = draft_model_config.get_num_layers(
411
+ vllm_config.parallel_config
412
+ )
413
+ logger.info("EAGLE detected %d extra layer(s)", num_mtp_layers)
414
+ except Exception:
415
+ logger.info(
416
+ "EAGLE detected, but failed to get the number of extra layers"
417
+ "falling back to 1"
418
+ )
419
+ num_mtp_layers = 1
420
return num_mtp_layers
421
422
@@ -1208,6 +1222,10 @@ class LMCacheConnectorV1Impl:
1222
if the CacheManager this allocated blocks for us.
1223
"""
1224
1225
+ # Clear local status in lookup client when a new request is
1226
+ # successfully scheduled.
1227
+ self.lookup_client.clear_lookup_status(request.request_id)
1228
+
1229
kv_transfer_params = (
1230
request.kv_transfer_params
1231
if hasattr(request, "kv_transfer_params")