| 1 | #!/bin/bash |
| 2 | # SPDX-License-Identifier: Apache-2.0 |
| 3 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 4 | # |
| 5 | # CPU offloading with LMCache in multi-process (MP) mode. |
| 6 | # |
| 7 | # In MP mode, LMCache runs as a standalone server process (`lmcache server`) |
| 8 | # that owns the KV cache storage. One or more vLLM instances connect to it via |
| 9 | # the `LMCacheMPConnector`. This is the recommended way to run LMCache for |
| 10 | # distributed KV storage and for sharing KV cache across vLLM instances. |
| 11 | # |
| 12 | # vLLM ships a built-in shortcut for this setup: pass `--kv-offloading-backend |
| 13 | # lmcache` together with `--kv-offloading-size <GiB>` and vLLM wires up the |
| 14 | # `LMCacheMPConnector` for you (it defaults to the LMCache server at |
| 15 | # tcp://localhost:5555, matching the `lmcache server` default). |
| 16 | # |
| 17 | # Requires `lmcache` to be installed (`pip install lmcache`). |
| 18 | # Learn more: https://docs.lmcache.ai |
| 19 | set -euo pipefail |
| 20 | |
| 21 | MODEL=${MODEL:-meta-llama/Meta-Llama-3.1-8B-Instruct} |
| 22 | |
| 23 | # 1. Launch the standalone LMCache server (binds tcp://localhost:5555 by |
| 24 | # default). `--l1-size-gb` sets the CPU memory budget for the L1 cache. |
| 25 | echo "Starting LMCache server..." |
| 26 | lmcache server --host localhost --port 5555 --l1-size-gb 5 & |
| 27 | LMCACHE_SERVER_PID=$! |
| 28 | trap 'kill $LMCACHE_SERVER_PID 2>/dev/null || true' EXIT |
| 29 | |
| 30 | # 2. Launch vLLM and offload KV cache to the LMCache server. |
| 31 | # The MP connector currently requires the non-hybrid KV cache manager. |
| 32 | echo "Starting vLLM server with LMCache MP offloading..." |
| 33 | vllm serve "$MODEL" \ |
| 34 | --port 8000 \ |
| 35 | --kv-offloading-size 5 \ |
| 36 | --kv-offloading-backend lmcache \ |
| 37 | --disable-hybrid-kv-cache-manager |
| 38 | |
| 39 | # Equivalent explicit configuration (instead of the two flags above): |
| 40 | # --kv-transfer-config \ |
| 41 | # '{"kv_connector":"LMCacheMPConnector","kv_role":"kv_both", |
| 42 | # "kv_connector_extra_config":{"lmcache.mp.host":"tcp://localhost", |
| 43 | # "lmcache.mp.port":5555}}' |