| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | from __future__ import annotations |
| 4 | |
| 5 | import math |
| 6 | from dataclasses import dataclass |
| 7 | from typing import TYPE_CHECKING |
| 8 | |
| 9 | from vllm.utils.network_utils import make_zmq_path |
| 10 | |
| 11 | if TYPE_CHECKING: |
| 12 | from vllm.config import VllmConfig |
| 13 | |
| 14 | # Scheduler and Worker must agree on when a remote reservation becomes stale. |
| 15 | _RESERVATION_TTL_SECONDS = 300 |
| 16 | |
| 17 | |
| 18 | def _positive_int(name: str, value: object) -> int: |
| 19 | message = f"ECMooncakeConnector requires {name} to be a positive integer." |
| 20 | if isinstance(value, bool): |
| 21 | raise ValueError(message) |
| 22 | if isinstance(value, int): |
| 23 | result = value |
| 24 | elif isinstance(value, float) and math.isfinite(value) and value.is_integer(): |
| 25 | result = int(value) |
| 26 | elif isinstance(value, str): |
| 27 | try: |
| 28 | result = int(value) |
| 29 | except ValueError as error: |
| 30 | raise ValueError(message) from error |
| 31 | else: |
| 32 | raise ValueError(message) |
| 33 | if result <= 0: |
| 34 | raise ValueError(f"ECMooncakeConnector requires {name} > 0.") |
| 35 | return result |
| 36 | |
| 37 | |
| 38 | def _positive_float(name: str, value: object) -> float: |
| 39 | message = f"ECMooncakeConnector requires {name} > 0." |
| 40 | if isinstance(value, bool): |
| 41 | raise ValueError(message) |
| 42 | try: |
| 43 | result = float(value) # type: ignore[arg-type] |
| 44 | except (TypeError, ValueError) as error: |
| 45 | raise ValueError(message) from error |
| 46 | if not math.isfinite(result) or result <= 0: |
| 47 | raise ValueError(message) |
| 48 | return result |
| 49 | |
| 50 | |
| 51 | @dataclass(frozen=True) |
| 52 | class MooncakeECConfig: |
| 53 | """Validated settings shared by the Scheduler and Worker roles. |
| 54 | |
| 55 | ``control_port`` is the first TP-shard port after the DP offset; |
| 56 | ``control_addr`` targets that shard, which advertises the full topology. |
| 57 | ``control_host`` is the address this instance both advertises and binds, |
| 58 | so reaching a Consumer means being routed to it rather than finding it on |
| 59 | every interface. |
| 60 | """ |
| 61 | |
| 62 | is_producer: bool |
| 63 | is_consumer: bool |
| 64 | protocol: str |
| 65 | buffer_device: str |
| 66 | control_host: str |
| 67 | control_port: int |
| 68 | control_addr: str |
| 69 | control_timeout_ms: int |
| 70 | push_wait_timeout_s: float |
| 71 | pool_size: int |
| 72 | cross_encoder_cache: bool = False |
| 73 | store_max_pending_items: int = 32 |
| 74 | store_max_pending_bytes: int = 2 * 1024**3 |
| 75 | store_read_buffer_bytes: int = 128 * 1024**2 |
| 76 | |
| 77 | @classmethod |
| 78 | def from_vllm_config(cls, vllm_config: VllmConfig) -> MooncakeECConfig: |
| 79 | parallel_config = vllm_config.parallel_config |
| 80 | ec_config = vllm_config.ec_transfer_config |
| 81 | assert ec_config is not None |
| 82 | |
| 83 | if ec_config.is_ec_producer: |
| 84 | if parallel_config.tensor_parallel_size > 1: |
| 85 | raise ValueError( |
| 86 | "ECMooncakeConnector producers require tensor_parallel_size=1." |
| 87 | ) |
| 88 | if parallel_config.pipeline_parallel_size > 1: |
| 89 | raise ValueError( |
| 90 | "ECMooncakeConnector producers do not support pipeline parallelism." |
| 91 | ) |
| 92 | if parallel_config.data_parallel_size > 1: |
| 93 | raise ValueError( |
| 94 | "ECMooncakeConnector producers require data_parallel_size=1." |
| 95 | ) |
| 96 | |
| 97 | registered_buffer_size = _positive_int( |
| 98 | "ec_buffer_size", ec_config.ec_buffer_size |
| 99 | ) |
| 100 | get = ec_config.get_from_extra_config |
| 101 | shared_reuse = get("cross_encoder_cache", False) |
| 102 | if not isinstance(shared_reuse, bool): |
| 103 | raise ValueError("cross_encoder_cache must be a boolean") |
| 104 | if shared_reuse: |
| 105 | if not ec_config.is_encode_only: |
| 106 | raise ValueError( |
| 107 | "cross_encoder_cache requires an Encoder-only producer" |
| 108 | ) |
| 109 | if not vllm_config.use_v2_model_runner: |
| 110 | raise ValueError("cross_encoder_cache requires Model Runner V2") |
| 111 | if vllm_config.lora_config is not None: |
| 112 | raise ValueError("cross_encoder_cache does not support LoRA") |
| 113 | mm_config = vllm_config.model_config.multimodal_config |
| 114 | if mm_config is not None and mm_config.mm_processor_cache_gb == 0: |
| 115 | raise ValueError( |
| 116 | "cross_encoder_cache requires mm_processor_cache_gb > 0 " |
| 117 | "to preserve content identifiers across Encoders and restarts" |
| 118 | ) |
| 119 | control_port = int(ec_config.ec_port) + ( |
| 120 | parallel_config.data_parallel_index * parallel_config.tensor_parallel_size |
| 121 | ) |
| 122 | highest_port = control_port + parallel_config.tensor_parallel_size - 1 |
| 123 | if not 1 <= control_port <= highest_port <= 65535: |
| 124 | raise ValueError("ECMooncakeConnector ec_port must be in 1..65535.") |
| 125 | |
| 126 | return cls( |
| 127 | is_producer=ec_config.is_ec_producer, |
| 128 | is_consumer=ec_config.is_ec_consumer, |
| 129 | protocol=str(get("mooncake_protocol", "rdma")), |
| 130 | buffer_device=str(ec_config.ec_buffer_device or "cuda").lower(), |
| 131 | control_host=str(ec_config.ec_ip), |
| 132 | control_port=control_port, |
| 133 | control_addr=make_zmq_path("tcp", ec_config.ec_ip, control_port), |
| 134 | control_timeout_ms=max( |
| 135 | 1, |
| 136 | math.ceil( |
| 137 | _positive_float("control_timeout_s", get("control_timeout_s", 30)) |
| 138 | * 1000 |
| 139 | ), |
| 140 | ), |
| 141 | push_wait_timeout_s=_positive_float( |
| 142 | "push_wait_timeout_s", get("push_wait_timeout_s", 60) |
| 143 | ), |
| 144 | pool_size=registered_buffer_size, |
| 145 | cross_encoder_cache=shared_reuse, |
| 146 | store_max_pending_items=_positive_int( |
| 147 | "store_max_pending_items", get("store_max_pending_items", 32) |
| 148 | ), |
| 149 | store_max_pending_bytes=_positive_int( |
| 150 | "store_max_pending_bytes", get("store_max_pending_bytes", 2 * 1024**3) |
| 151 | ), |
| 152 | store_read_buffer_bytes=_positive_int( |
| 153 | "store_read_buffer_bytes", get("store_read_buffer_bytes", 128 * 1024**2) |
| 154 | ), |
| 155 | ) |