| 1 | #!/bin/bash |
| 2 | # ============================================================================= |
| 3 | # cluster.sh — single, generic config for the vLLM disaggregated (P/D) launcher. |
| 4 | # ----------------------------------------------------------------------------- |
| 5 | # This script is *sourced* by vllm_disagg.sh. |
| 6 | # |
| 7 | # Every value is `${VAR:-default}`, so the environment always wins: |
| 8 | # environment variable > built-in default below |
| 9 | # |
| 10 | # So you can override anything inline: |
| 11 | # PREFILL_IP=10.0.0.1 DECODE_IP=10.0.0.2 ./vllm_disagg.sh prefill |
| 12 | # |
| 13 | # Site-specific values (model dir, IPs, NIC list, partition) are the defaults |
| 14 | # in the "site defaults" sections below — edit those for a new cluster. |
| 15 | # Model-SPECIFIC perf flags live in models.yaml, NOT here. |
| 16 | # ============================================================================= |
| 17 | |
| 18 | _CLUSTER_SH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 19 | |
| 20 | # ----------------------------------------------------------------- model / mode |
| 21 | # MODEL_NAME indexes into models.yaml; MODEL_DIR is the parent dir holding it. |
| 22 | # MODEL_PATH (resolved by the launcher) = ${MODEL_DIR}/${MODEL_NAME}. [site] |
| 23 | export MODEL_NAME="${MODEL_NAME:-DeepSeek-V3}" |
| 24 | export MODEL_DIR="${MODEL_DIR:-/data/models2}" |
| 25 | |
| 26 | # Shared NFS root (5 TB, visible on every node): model weights + per-run logs. [site] |
| 27 | export SHARED_MOUNT="${SHARED_MOUNT:-/data}" |
| 28 | |
| 29 | # Parallelism mode (launcher derives PARALLEL_MODE tp|ep from this): |
| 30 | # WIDE_EP_MODE=0 tp : each node is an independent TP server (TP8, 1P1D) |
| 31 | # WIDE_EP_MODE=1 ep : data-parallel + expert-parallel across xP/yD nodes (wideep) |
| 32 | export WIDE_EP_MODE="${WIDE_EP_MODE:-0}" |
| 33 | |
| 34 | # ----------------------------------------------------------------- topology |
| 35 | # xP prefill nodes + yD decode nodes. IPADDRS is the ordered, comma-separated |
| 36 | # node list (prefill IPs first, then decode IPs). NODE_RANK is this node's global |
| 37 | # 0-based rank; under SLURM it defaults to $SLURM_PROCID. Leave IPADDRS empty to |
| 38 | # use the PREFILL_IP/DECODE_IP fallback defaults below (1P1D only). |
| 39 | export xP="${xP:-1}" |
| 40 | export yD="${yD:-1}" |
| 41 | export IPADDRS="${IPADDRS:-}" |
| 42 | export NODE_RANK="${NODE_RANK:-${SLURM_PROCID:-}}" |
| 43 | export PREFILL_IP="${PREFILL_IP:-10.0.0.1}" |
| 44 | export DECODE_IP="${DECODE_IP:-10.0.0.2}" |
| 45 | |
| 46 | # Per-node GPU count and TP degree |
| 47 | export GPUS_PER_NODE="${GPUS_PER_NODE:-8}" |
| 48 | export TP_SIZE="${TP_SIZE:-${GPUS_PER_NODE}}" |
| 49 | |
| 50 | # ----------------------------------------------------------------- ports |
| 51 | # TP-mode (WIDE_EP_MODE=0) server ports. |
| 52 | export PREFILL_PORT="${PREFILL_PORT:-8100}" |
| 53 | export DECODE_PORT="${DECODE_PORT:-8200}" |
| 54 | |
| 55 | # EP-mode (WIDE_EP_MODE=1) ports: API serve port, DP RPC port, KV transfer port, and |
| 56 | # per-node MoRIIO local ping port (must differ from PROXY_PING_PORT). |
| 57 | export SERVE_PORT="${SERVE_PORT:-20005}" |
| 58 | export RPC_PORT="${RPC_PORT:-13345}" |
| 59 | export KV_PORT="${KV_PORT:-9711}" |
| 60 | export LOCAL_PING_PORT="${LOCAL_PING_PORT:-61555}" |
| 61 | |
| 62 | # MoRIIO proxy: HTTP port clients/benchmark hit, plus the connector control ports. |
| 63 | # PROXY_PING_PORT MUST be 36367 — the proxy hardcodes its zmq service-discovery |
| 64 | # socket on that port; prefill/decode register to PROXY_IP:PROXY_PING_PORT. |
| 65 | export PROXY_IP="${PROXY_IP:-${PREFILL_IP}}" |
| 66 | export PROXY_PORT="${PROXY_PORT:-10001}" |
| 67 | export PROXY_PING_PORT="${PROXY_PING_PORT:-36367}" |
| 68 | export HANDSHAKE_PORT="${HANDSHAKE_PORT:-6301}" |
| 69 | export NOTIFY_PORT="${NOTIFY_PORT:-61005}" |
| 70 | |
| 71 | export PROXY_SCRIPT="${PROXY_SCRIPT:-/app/vllm/examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py}" |
| 72 | |
| 73 | # MoRIIO KV transfer direction (injected into --kv-transfer-config by the launcher): |
| 74 | # 0 -> omit read_mode (default; MoRIIO write mode: prefill pushes to decode) |
| 75 | # 1 -> "read_mode": true (decode pulls KV from prefill; matches upstream disagg) |
| 76 | export MORIIO_READ_MODE="${MORIIO_READ_MODE:-0}" |
| 77 | |
| 78 | # ----------------------------------------------------------------- router / gateway |
| 79 | # Selection for client (bench/accuracy) traffic: |
| 80 | # proxy -> the in-container MoRIIO proxy started by the launcher |
| 81 | # vllm-router -> an external `vllm/vllm-router` container started by the SLURM job |
| 82 | # on the rank-0 node (default) |
| 83 | # Both use the SAME MoRIIO discovery mechanism (prefill/decode register to |
| 84 | # PROXY_IP:PROXY_PING_PORT=36367); only the client HTTP front door differs. |
| 85 | export ROUTER_TYPE="${ROUTER_TYPE:-vllm-router}" |
| 86 | export ROUTER_PORT="${ROUTER_PORT:-30000}" |
| 87 | export ROUTER_POLICY="${ROUTER_POLICY:-round_robin}" |
| 88 | export VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly}" |
| 89 | # Single client-facing port bench/accuracy target: the router port when routing, |
| 90 | # else the proxy port. Env override always wins. |
| 91 | if [[ "${ROUTER_TYPE}" == "vllm-router" ]]; then |
| 92 | export GATEWAY_PORT="${GATEWAY_PORT:-${ROUTER_PORT}}" |
| 93 | else |
| 94 | export GATEWAY_PORT="${GATEWAY_PORT:-${PROXY_PORT}}" |
| 95 | fi |
| 96 | |
| 97 | # Where per-run logs / benchmark results are written. A $SLURM_JOB_ID subdir is |
| 98 | # appended so each CI run is self-scoped (falls back to 'local' off-SLURM). [site] |
| 99 | _LOG_BASE="${LOG_BASE:-/data/${USER:-$(id -un)}/disagg_logs}" |
| 100 | export LOG_PATH="${LOG_PATH:-${_LOG_BASE}/${SLURM_JOB_ID:-local}}" |
| 101 | |
| 102 | # ----------------------------------------------------------------- vLLM runtime |
| 103 | # Engine/platform/transport-level env (NOT model-specific). Model-architecture |
| 104 | # AITER kernel toggles live in models.yaml under each model's `env:` block. |
| 105 | export VLLM_USE_V1="${VLLM_USE_V1:-1}" |
| 106 | export HSA_NO_SCRATCH_RECLAIM="${HSA_NO_SCRATCH_RECLAIM:-1}" |
| 107 | |
| 108 | #export HF_HUB_OFFLINE="${HF_HUB_OFFLINE:-1}" |
| 109 | #export TRANSFORMERS_OFFLINE="${TRANSFORMERS_OFFLINE:-1}" |
| 110 | # |
| 111 | #export HOME=/tmp |
| 112 | export HF_HOME="${HF_HOME:-/tmp/hf_home}" |
| 113 | export XDG_CACHE_HOME="${XDG_CACHE_HOME:-/tmp/.cache}" |
| 114 | export VLLM_ENGINE_READY_TIMEOUT_S="${VLLM_ENGINE_READY_TIMEOUT_S:-3600}" |
| 115 | |
| 116 | # ----------------------------------------------------------------- scale |
| 117 | # DP/EP group formation timeout across nodes (seconds). |
| 118 | export DISTRIBUTED_TIMEOUT_SECONDS="${DISTRIBUTED_TIMEOUT_SECONDS:-7200}" |
| 119 | |
| 120 | # ----------------------------------------------------------------- RDMA / NCCL |
| 121 | # AMD Pensando AINIC RoCE fabric: 8 NICs exposed as ionic_0..7 (netdevs eth2..9), |
| 122 | # each rail on its own /24. GID index 1 + traffic class 104 |
| 123 | _IB_DEVICES="${IB_DEVICES:-ionic_0,ionic_1,ionic_2,ionic_3,ionic_4,ionic_5,ionic_6,ionic_7}" |
| 124 | _IB_GID_INDEX="${NCCL_IB_GID_INDEX:-1}" |
| 125 | export IB_DEVICES="${IB_DEVICES:-${_IB_DEVICES}}" |
| 126 | export NCCL_IB_HCA="${NCCL_IB_HCA:-${_IB_DEVICES}}" |
| 127 | export NCCL_IB_GID_INDEX="${NCCL_IB_GID_INDEX:-${_IB_GID_INDEX}}" |
| 128 | export NCCL_IB_DISABLE="${NCCL_IB_DISABLE:-0}" |
| 129 | # In-box RCCL net transport (no external plugin); pin bootstrap to the VPC iface. |
| 130 | export NCCL_NET_PLUGIN="${NCCL_NET_PLUGIN:-none}" |
| 131 | export NCCL_SOCKET_IFNAME="${NCCL_SOCKET_IFNAME:-eth0}" |
| 132 | export GLOO_SOCKET_IFNAME="${GLOO_SOCKET_IFNAME:-${NCCL_SOCKET_IFNAME}}" |
| 133 | export NCCL_CROSS_NIC="${NCCL_CROSS_NIC:-0}" |
| 134 | export NCCL_PXN_DISABLE="${NCCL_PXN_DISABLE:-0}" |
| 135 | export NCCL_NET_DISABLE_INTRA="${NCCL_NET_DISABLE_INTRA:-1}" |
| 136 | export NCCL_IB_TC="${NCCL_IB_TC:-104}" |
| 137 | export NCCL_IB_FIFO_TC="${NCCL_IB_FIFO_TC:-192}" |
| 138 | export NCCL_IB_QPS_PER_CONNECTION="${NCCL_IB_QPS_PER_CONNECTION:-1}" |
| 139 | export NCCL_IB_TIMEOUT="${NCCL_IB_TIMEOUT:-22}" |
| 140 | export NCCL_IB_RETRY_CNT="${NCCL_IB_RETRY_CNT:-12}" |
| 141 | |
| 142 | # MoRI uses the same NIC set as NCCL. |
| 143 | export MORI_RDMA_DEVICES="${MORI_RDMA_DEVICES:-${_IB_DEVICES}}" |
| 144 | export MORI_IB_GID_INDEX="${MORI_IB_GID_INDEX:-${_IB_GID_INDEX}}" |
| 145 | export MORI_SHMEM_HEAP_SIZE="${MORI_SHMEM_HEAP_SIZE:-16G}" |
| 146 | |
| 147 | # Pin to gfx950 to avoid jit compilation failures with other archs on this cluster. |
| 148 | export MORI_GPU_ARCHS="gfx950" |
| 149 | |
| 150 | # ----------------------------------------------------------------- benchmark |
| 151 | export BENCHMARK_COMBINATIONS="${BENCHMARK_COMBINATIONS:-1024/128 2048/128}" |
| 152 | export BENCHMARK_CON="${BENCHMARK_CON:-32 64}" |
| 153 | export NUM_PROMPTS_FACTOR="${NUM_PROMPTS_FACTOR:-2}" |
| 154 | export BENCHMARK_MIN_PROMPTS="${BENCHMARK_MIN_PROMPTS:-32}" |
| 155 | |
| 156 | # ----------------------------------------------------------------- accuracy |
| 157 | # `accuracy` role runs lm_eval (local-completions backend) against the proxy. |
| 158 | export ACCURACY_TASKS="${ACCURACY_TASKS:-gsm8k}" |
| 159 | export ACCURACY_NUM_CONCURRENT="${ACCURACY_NUM_CONCURRENT:-64}" |
| 160 | export ACCURACY_MAX_RETRIES="${ACCURACY_MAX_RETRIES:-3}" |
| 161 | export ACCURACY_METRIC="${ACCURACY_METRIC:-exact_match}" |
| 162 | export ACCURACY_THRESHOLD="${ACCURACY_THRESHOLD:-0.90}" |
| 163 | |
| 164 | # ----------------------------------------------------------------- SLURM (submit) |
| 165 | # Used by run-slurm-disagg-test.sh on the login node (harmless to export here). [site] |
| 166 | export SLURM_PARTITION="${SLURM_PARTITION:-default}" |