@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0590ec3fd9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Core] Implement disagg prefill by StatelessProcessGroup (#10502)

This PR provides initial support for single-node disaggregated prefill in 1P1D scenario. Signed-off-by: KuntaiDu <kuntai@uchicago.edu> Co-authored-by: ApostaC <yihua98@uchicago.edu> Co-authored-by: YaoJiayi <120040070@link.cuhk.edu.cn>

Kuntai Du committed Dec 1, 2024 at 19:01 UTC 0590ec3fd9857063c43c80df281e24c16c51b2ec
33 files changed +2525 -21
.buildkite/test-pipeline.yaml
+4
@@ -430,6 +430,9 @@ steps:
430 - vllm/model_executor/models/
431 - tests/distributed/
432 - vllm/compilation
433 + - vllm/worker/worker_base.py
434 + - vllm/worker/worker.py
435 + - vllm/worker/model_runner.py
436 commands:
437 - pytest -v -s ./compile/test_basic_correctness.py
438 - pytest -v -s ./compile/test_wrapper.py
@@ -443,6 +446,7 @@ steps:
446 - pip install -e ./plugins/vllm_add_dummy_model
447 - pytest -v -s distributed/test_distributed_oot.py
448 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
449 + - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s kv_transfer/disagg_test.py
450
451 - label: Multi-step Tests (4 GPUs) # 36min
452 working_dir: "/vllm-workspace/tests"
benchmarks/disagg_benchmarks/disagg_overhead_benchmark.sh new
+144
@@ -0,0 +1,144 @@
1 +#!/bin/bash
2 +
3 +# benchmark the overhead of disaggregated prefill.
4 +# methodology:
5 +# - send all request to prefill vLLM instance. It will buffer KV cache.
6 +# - then send all request to decode instance.
7 +# - The TTFT of decode instance is the overhead.
8 +
9 +set -ex
10 +
11 +kill_gpu_processes() {
12 + # kill all processes on GPU.
13 + pkill -f pt_main_thread
14 + sleep 10
15 +
16 + # remove vllm config file
17 + rm -rf ~/.config/vllm
18 +
19 + # Print the GPU memory usage
20 + # so that we know if all GPU processes are killed.
21 + gpu_memory_usage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i 0)
22 + # The memory usage should be 0 MB.
23 + echo "GPU 0 Memory Usage: $gpu_memory_usage MB"
24 +}
25 +
26 +wait_for_server() {
27 + # wait for vllm server to start
28 + # return 1 if vllm server crashes
29 + local port=$1
30 + timeout 1200 bash -c "
31 + until curl -s localhost:${port}/v1/completions > /dev/null; do
32 + sleep 1
33 + done" && return 0 || return 1
34 +}
35 +
36 +
37 +benchmark() {
38 +
39 + export VLLM_LOGGING_LEVEL=DEBUG
40 + export VLLM_HOST_IP=$(hostname -I | awk '{print $1}')
41 +
42 + # compare chunked prefill with disaggregated prefill
43 +
44 + results_folder="./results"
45 + model="meta-llama/Meta-Llama-3.1-8B-Instruct"
46 + dataset_name="sonnet"
47 + dataset_path="../sonnet_4x.txt"
48 + num_prompts=10
49 + qps=$1
50 + prefix_len=50
51 + input_len=2048
52 + output_len=$2
53 +
54 +
55 + CUDA_VISIBLE_DEVICES=0 python3 \
56 + -m vllm.entrypoints.openai.api_server \
57 + --model meta-llama/Meta-Llama-3.1-8B-Instruct \
58 + --port 8100 \
59 + --max-model-len 10000 \
60 + --gpu-memory-utilization 0.6 \
61 + --kv-transfer-config \
62 + '{"kv_connector":"PyNcclConnector","kv_role":"kv_producer","kv_rank":0,"kv_parallel_size":2,"kv_buffer_size":5e9}' &
63 +
64 +
65 + CUDA_VISIBLE_DEVICES=1 python3 \
66 + -m vllm.entrypoints.openai.api_server \
67 + --model meta-llama/Meta-Llama-3.1-8B-Instruct \
68 + --port 8200 \
69 + --max-model-len 10000 \
70 + --gpu-memory-utilization 0.6 \
71 + --kv-transfer-config \
72 + '{"kv_connector":"PyNcclConnector","kv_role":"kv_consumer","kv_rank":1,"kv_parallel_size":2,"kv_buffer_size":5e9}' &
73 +
74 + wait_for_server 8100
75 + wait_for_server 8200
76 +
77 + # let the prefill instance finish prefill
78 + python3 ../benchmark_serving.py \
79 + --backend vllm \
80 + --model $model \
81 + --dataset-name $dataset_name \
82 + --dataset-path $dataset_path \
83 + --sonnet-input-len $input_len \
84 + --sonnet-output-len "$output_len" \
85 + --sonnet-prefix-len $prefix_len \
86 + --num-prompts $num_prompts \
87 + --port 8100 \
88 + --save-result \
89 + --result-dir $results_folder \
90 + --result-filename disagg_prefill_2xtp4.json \
91 + --request-rate "inf"
92 +
93 +
94 + # send the request to decode.
95 + # The TTFT of this command will be the overhead of disagg prefill impl.
96 + python3 ../benchmark_serving.py \
97 + --backend vllm \
98 + --model $model \
99 + --dataset-name $dataset_name \
100 + --dataset-path $dataset_path \
101 + --sonnet-input-len $input_len \
102 + --sonnet-output-len "$output_len" \
103 + --sonnet-prefix-len $prefix_len \
104 + --num-prompts $num_prompts \
105 + --port 8200 \
106 + --save-result \
107 + --result-dir $results_folder \
108 + --result-filename disagg_prefill_2xtp4.json \
109 + --request-rate "$qps"
110 + kill_gpu_processes
111 +
112 +}
113 +
114 +
115 +main() {
116 +
117 + (which wget && which curl) || (apt-get update && apt-get install -y wget curl)
118 + (which jq) || (apt-get -y install jq)
119 + (which socat) || (apt-get -y install socat)
120 +
121 + pip install quart httpx
122 +
123 + cd "$(dirname "$0")"
124 +
125 + cd ..
126 + # create sonnet-4x.txt
127 + echo "" > sonnet_4x.txt
128 + for _ in {1..4}
129 + do
130 + cat sonnet.txt >> sonnet_4x.txt
131 + done
132 + cd disagg_benchmarks
133 +
134 + rm -rf results
135 + mkdir results
136 +
137 + default_qps=1
138 + default_output_len=1
139 + benchmark $default_qps $default_output_len
140 +
141 +}
142 +
143 +
144 +main "$@"
benchmarks/disagg_benchmarks/disagg_performance_benchmark.sh new
+164
@@ -0,0 +1,164 @@
1 +#!/bin/bash
2 +
3 +# Requirement: 8x H100 GPUs.
4 +
5 +
6 +# Model: neuralmagic/Meta-Llama-3-70B-Instruct-FP8-KV
7 +# Query: 2048 input tokens, 11 output tokens, QPS 4, 500 requests
8 +# Resource: 8x H100
9 +# Approaches:
10 +# 1. Chunked prefill: 1 vllm instance with tp=8
11 +# 2. Chunked prefill: 2 vllm instance with tp=4, equivalent to 1 tp=4 instance with QPS 4
12 +# 3. Disaggregated prefill: 1 prefilling instance and 1 decoding instance
13 +# Prefilling instance: max_output_token=1
14 +# Decoding instance: force the input tokens be the same across requests to bypass prefilling
15 +
16 +set -ex
17 +
18 +kill_gpu_processes() {
19 + # kill all processes on GPU.
20 + pgrep pt_main_thread | xargs -r kill -9
21 + pgrep python3 | xargs -r kill -9
22 + for port in 8000 8100 8200; do lsof -t -i:$port | xargs -r kill -9; done
23 + sleep 1
24 +}
25 +
26 +wait_for_server() {
27 + # wait for vllm server to start
28 + # return 1 if vllm server crashes
29 + local port=$1
30 + timeout 1200 bash -c "
31 + until curl -s localhost:${port}/v1/completions > /dev/null; do
32 + sleep 1
33 + done" && return 0 || return 1
34 +}
35 +
36 +
37 +launch_chunked_prefill() {
38 + model="meta-llama/Meta-Llama-3.1-8B-Instruct"
39 + # disagg prefill
40 + CUDA_VISIBLE_DEVICES=0 python3 \
41 + -m vllm.entrypoints.openai.api_server \
42 + --model $model \
43 + --port 8100 \
44 + --max-model-len 10000 \
45 + --enable-chunked-prefill \
46 + --gpu-memory-utilization 0.6 &
47 + CUDA_VISIBLE_DEVICES=1 python3 \
48 + -m vllm.entrypoints.openai.api_server \
49 + --model $model \
50 + --port 8200 \
51 + --max-model-len 10000 \
52 + --enable-chunked-prefill \
53 + --gpu-memory-utilization 0.6 &
54 + wait_for_server 8100
55 + wait_for_server 8200
56 + python3 round_robin_proxy.py &
57 + sleep 1
58 +}
59 +
60 +
61 +launch_disagg_prefill() {
62 + model="meta-llama/Meta-Llama-3.1-8B-Instruct"
63 + # disagg prefill
64 + CUDA_VISIBLE_DEVICES=0 python3 \
65 + -m vllm.entrypoints.openai.api_server \
66 + --model $model \
67 + --port 8100 \
68 + --max-model-len 10000 \
69 + --gpu-memory-utilization 0.6 \
70 + --kv-transfer-config \
71 + '{"kv_connector":"PyNcclConnector","kv_role":"kv_producer","kv_rank":0,"kv_parallel_size":2,"kv_buffer_size":5e9}' &
72 +
73 + CUDA_VISIBLE_DEVICES=1 python3 \
74 + -m vllm.entrypoints.openai.api_server \
75 + --model $model \
76 + --port 8200 \
77 + --max-model-len 10000 \
78 + --gpu-memory-utilization 0.6 \
79 + --kv-transfer-config \
80 + '{"kv_connector":"PyNcclConnector","kv_role":"kv_consumer","kv_rank":1,"kv_parallel_size":2,"kv_buffer_size":5e9}' &
81 +
82 + wait_for_server 8100
83 + wait_for_server 8200
84 + python3 disagg_prefill_proxy_server.py &
85 + sleep 1
86 +}
87 +
88 +
89 +benchmark() {
90 + results_folder="./results"
91 + model="meta-llama/Meta-Llama-3.1-8B-Instruct"
92 + dataset_name="sonnet"
93 + dataset_path="../sonnet_4x.txt"
94 + num_prompts=100
95 + qps=$1
96 + prefix_len=50
97 + input_len=1024
98 + output_len=$2
99 + tag=$3
100 +
101 + python3 ../benchmark_serving.py \
102 + --backend vllm \
103 + --model $model \
104 + --dataset-name $dataset_name \
105 + --dataset-path $dataset_path \
106 + --sonnet-input-len $input_len \
107 + --sonnet-output-len "$output_len" \
108 + --sonnet-prefix-len $prefix_len \
109 + --num-prompts $num_prompts \
110 + --port 8000 \
111 + --save-result \
112 + --result-dir $results_folder \
113 + --result-filename "$tag"-qps-"$qps".json \
114 + --request-rate "$qps"
115 +
116 + sleep 2
117 +
118 +}
119 +
120 +
121 +main() {
122 +
123 + (which wget && which curl) || (apt-get update && apt-get install -y wget curl)
124 + (which jq) || (apt-get -y install jq)
125 + (which socat) || (apt-get -y install socat)
126 +
127 + pip install quart httpx matplotlib aiohttp
128 +
129 + cd "$(dirname "$0")"
130 +
131 + cd ..
132 + # create sonnet-4x.txt so that we can sample 2048 tokens for input
133 + echo "" > sonnet_4x.txt
134 + for _ in {1..4}
135 + do
136 + cat sonnet.txt >> sonnet_4x.txt
137 + done
138 + cd disagg_benchmarks
139 +
140 + rm -rf results
141 + mkdir results
142 +
143 + default_output_len=6
144 +
145 + export VLLM_HOST_IP=$(hostname -I | awk '{print $1}')
146 +
147 + launch_chunked_prefill
148 + for qps in 2 4 6 8; do
149 + benchmark $qps $default_output_len chunked_prefill
150 + done
151 + kill_gpu_processes
152 +
153 + launch_disagg_prefill
154 + for qps in 2 4 6 8; do
155 + benchmark $qps $default_output_len disagg_prefill
156 + done
157 + kill_gpu_processes
158 +
159 + python3 visualize_benchmark_results.py
160 +
161 +}
162 +
163 +
164 +main "$@"
benchmarks/disagg_benchmarks/disagg_prefill_proxy_server.py new
+61
@@ -0,0 +1,61 @@
1 +import os
2 +
3 +import aiohttp
4 +from quart import Quart, make_response, request
5 +
6 +AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=6 * 60 * 60)
7 +
8 +app = Quart(__name__)
9 +
10 +
11 +async def forward_request(url, data):
12 + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
13 + headers = {
14 + "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"
15 + }
16 + async with session.post(url=url, json=data,
17 + headers=headers) as response:
18 + if response.status == 200:
19 + # if response.headers.get('Transfer-Encoding') == 'chunked':
20 + if True:
21 + async for chunk_bytes in response.content.iter_chunked(
22 + 1024):
23 + yield chunk_bytes
24 + else:
25 + content = await response.read()
26 + yield content
27 +
28 +
29 +@app.route('/v1/completions', methods=['POST'])
30 +async def handle_request():
31 + try:
32 + original_request_data = await request.get_json()
33 +
34 + prefill_request = original_request_data.copy()
35 + # change max_tokens = 1 to let it only do prefill
36 + prefill_request['max_tokens'] = 1
37 +
38 + # finish prefill
39 + async for _ in forward_request('http://localhost:8100/v1/completions',
40 + prefill_request):
41 + continue
42 +
43 + # return decode
44 + generator = forward_request('http://localhost:8200/v1/completions',
45 + original_request_data)
46 + response = await make_response(generator)
47 + response.timeout = None
48 +
49 + return response
50 +
51 + except Exception as e:
52 + import sys
53 + import traceback
54 + exc_info = sys.exc_info()
55 + print("Error occurred in disagg prefill proxy server")
56 + print(e)
57 + print("".join(traceback.format_exception(*exc_info)))
58 +
59 +
60 +if __name__ == '__main__':
61 + app.run(port=8000)
benchmarks/disagg_benchmarks/round_robin_proxy.py new
+60
@@ -0,0 +1,60 @@
1 +import asyncio
2 +import itertools
3 +
4 +import aiohttp
5 +from aiohttp import web
6 +
7 +
8 +class RoundRobinProxy:
9 +
10 + def __init__(self, target_ports):
11 + self.target_ports = target_ports
12 + self.port_cycle = itertools.cycle(self.target_ports)
13 +
14 + async def handle_request(self, request):
15 + target_port = next(self.port_cycle)
16 + target_url = f"http://localhost:{target_port}{request.path_qs}"
17 +
18 + async with aiohttp.ClientSession() as session:
19 + try:
20 + # Forward the request
21 + async with session.request(
22 + method=request.method,
23 + url=target_url,
24 + headers=request.headers,
25 + data=request.content,
26 + ) as response:
27 + # Start sending the response
28 + resp = web.StreamResponse(status=response.status,
29 + headers=response.headers)
30 + await resp.prepare(request)
31 +
32 + # Stream the response content
33 + async for chunk in response.content.iter_any():
34 + await resp.write(chunk)
35 +
36 + await resp.write_eof()
37 + return resp
38 +
39 + except Exception as e:
40 + return web.Response(text=f"Error: {str(e)}", status=500)
41 +
42 +
43 +async def main():
44 + proxy = RoundRobinProxy([8100, 8200])
45 + app = web.Application()
46 + app.router.add_route('*', '/{path:.*}', proxy.handle_request)
47 +
48 + runner = web.AppRunner(app)
49 + await runner.setup()
50 + site = web.TCPSite(runner, 'localhost', 8000)
51 + await site.start()
52 +
53 + print("Proxy server started on http://localhost:8000")
54 +
55 + # Keep the server running
56 + await asyncio.Event().wait()
57 +
58 +
59 +if __name__ == '__main__':
60 + asyncio.run(main())
benchmarks/disagg_benchmarks/visualize_benchmark_results.py new
+46
@@ -0,0 +1,46 @@
1 +import json
2 +
3 +import matplotlib.pyplot as plt
4 +import pandas as pd
5 +
6 +if __name__ == "__main__":
7 +
8 + data = []
9 + for name in ['disagg_prefill', 'chunked_prefill']:
10 + for qps in [2, 4, 6, 8]:
11 + with open(f"results/{name}-qps-{qps}.json") as f:
12 + x = json.load(f)
13 + x['name'] = name
14 + x['qps'] = qps
15 + data.append(x)
16 +
17 + df = pd.DataFrame.from_dict(data)
18 + dis_df = df[df['name'] == 'disagg_prefill']
19 + chu_df = df[df['name'] == 'chunked_prefill']
20 +
21 + plt.style.use('bmh')
22 + plt.rcParams['font.size'] = 20
23 +
24 + for key in [
25 + 'mean_ttft_ms', 'median_ttft_ms', 'p99_ttft_ms', 'mean_itl_ms',
26 + 'median_itl_ms', 'p99_itl_ms'
27 + ]:
28 +
29 + fig, ax = plt.subplots(figsize=(11, 7))
30 + plt.plot(dis_df['qps'],
31 + dis_df[key],
32 + label='disagg_prefill',
33 + marker='o',
34 + linewidth=4)
35 + plt.plot(chu_df['qps'],
36 + chu_df[key],
37 + label='chunked_prefill',
38 + marker='o',
39 + linewidth=4)
40 + ax.legend()
41 +
42 + ax.set_xlabel('QPS')
43 + ax.set_ylabel(key)
44 + ax.set_ylim(bottom=0)
45 + fig.savefig(f'results/{key}.png')
46 + plt.close(fig)
examples/disaggregated_prefill.sh new
+109
@@ -0,0 +1,109 @@
1 +#!/bin/bash
2 +# This file demonstrates the example usage of disaggregated prefilling
3 +# We will launch 2 vllm instances (1 for prefill and 1 for decode),
4 +# and then transfer the KV cache between them.
5 +
6 +echo "🚧🚧 Warning: The usage of disaggregated prefill is experimental and subject to change 🚧🚧"
7 +sleep 1
8 +
9 +# Trap the SIGINT signal (triggered by Ctrl+C)
10 +trap 'cleanup' INT
11 +
12 +# Cleanup function
13 +cleanup() {
14 + echo "Caught Ctrl+C, cleaning up..."
15 + # Cleanup commands
16 + pgrep python | xargs kill -9
17 + pkill -f python
18 + echo "Cleanup complete. Exiting."
19 + exit 0
20 +}
21 +
22 +export VLLM_HOST_IP=$(hostname -I | awk '{print $1}')
23 +
24 +# install quart first -- required for disagg prefill proxy serve
25 +if python3 -c "import quart" &> /dev/null; then
26 + echo "Quart is already installed."
27 +else
28 + echo "Quart is not installed. Installing..."
29 + python3 -m pip install quart
30 +fi
31 +
32 +# a function that waits vLLM server to start
33 +wait_for_server() {
34 + local port=$1
35 + timeout 1200 bash -c "
36 + until curl -s localhost:${port}/v1/completions > /dev/null; do
37 + sleep 1
38 + done" && return 0 || return 1
39 +}
40 +
41 +
42 +# You can also adjust --kv-ip and --kv-port for distributed inference.
43 +
44 +# prefilling instance, which is the KV producer
45 +CUDA_VISIBLE_DEVICES=0 vllm serve meta-llama/Meta-Llama-3.1-8B-Instruct \
46 + --port 8100 \
47 + --max-model-len 100 \
48 + --gpu-memory-utilization 0.8 \
49 + --kv-transfer-config \
50 + '{"kv_connector":"PyNcclConnector","kv_role":"kv_producer","kv_rank":0,"kv_parallel_size":2}' &
51 +
52 +# decoding instance, which is the KV consumer
53 +CUDA_VISIBLE_DEVICES=1 vllm serve meta-llama/Meta-Llama-3.1-8B-Instruct \
54 + --port 8200 \
55 + --max-model-len 100 \
56 + --gpu-memory-utilization 0.8 \
57 + --kv-transfer-config \
58 + '{"kv_connector":"PyNcclConnector","kv_role":"kv_consumer","kv_rank":1,"kv_parallel_size":2}' &
59 +
60 +# wait until prefill and decode instances are ready
61 +wait_for_server 8100
62 +wait_for_server 8200
63 +
64 +# launch a proxy server that opens the service at port 8000
65 +# the workflow of this proxy:
66 +# - send the request to prefill vLLM instance (port 8100), change max_tokens
67 +# to 1
68 +# - after the prefill vLLM finishes prefill, send the request to decode vLLM
69 +# instance
70 +# NOTE: the usage of this API is subject to change --- in the future we will
71 +# introduce "vllm connect" to connect between prefill and decode instances
72 +python3 ../benchmarks/disagg_benchmarks/disagg_prefill_proxy_server.py &
73 +sleep 1
74 +
75 +# serve two example requests
76 +output1=$(curl -X POST -s http://localhost:8000/v1/completions \
77 +-H "Content-Type: application/json" \
78 +-d '{
79 +"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
80 +"prompt": "San Francisco is a",
81 +"max_tokens": 10,
82 +"temperature": 0
83 +}')
84 +
85 +output2=$(curl -X POST -s http://localhost:8000/v1/completions \
86 +-H "Content-Type: application/json" \
87 +-d '{
88 +"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
89 +"prompt": "Santa Clara is a",
90 +"max_tokens": 10,
91 +"temperature": 0
92 +}')
93 +
94 +
95 +# Cleanup commands
96 +pgrep python | xargs kill -9
97 +pkill -f python
98 +
99 +echo ""
100 +
101 +sleep 1
102 +
103 +# Print the outputs of the curl requests
104 +echo ""
105 +echo "Output of first request: $output1"
106 +echo "Output of second request: $output2"
107 +
108 +echo "🎉🎉 Successfully finished 2 test requests! 🎉🎉"
109 +echo ""
tests/kv_transfer/disagg_test.py new
+119
@@ -0,0 +1,119 @@
1 +import os
2 +import subprocess
3 +import sys
4 +import time
5 +from subprocess import Popen
6 +
7 +import pytest
8 +import requests
9 +import torch
10 +
11 +
12 +# Fixture to set up environment variables and teardown servers after tests
13 +@pytest.fixture(scope="module", autouse=True)
14 +def setup_servers():
15 + if torch.cuda.device_count() < 4:
16 + pytest.skip("Skipping test: fewer than 4 GPUs available")
17 +
18 + # Set up environment variables
19 + VLLM_HOST_IP = subprocess.check_output("hostname -I | awk '{print $1}'",
20 + shell=True).decode().strip()
21 + os.environ["VLLM_HOST_IP"] = VLLM_HOST_IP
22 +
23 + # Start prefill instance
24 + prefill_cmd = [
25 + sys.executable,
26 + "-m",
27 + "vllm.entrypoints.openai.api_server",
28 + "--model",
29 + "meta-llama/Meta-Llama-3.1-8B-Instruct",
30 + "--port",
31 + "8100",
32 + "--gpu-memory-utilization",
33 + "0.5",
34 + "--max-model-len",
35 + "1000",
36 + "--kv-transfer-config",
37 + '{"kv_connector":"PyNcclConnector","kv_role":"kv_producer",'\
38 + '"kv_rank":0,"kv_parallel_size":2}',
39 + ]
40 + prefill_env = os.environ.copy()
41 + prefill_env["CUDA_VISIBLE_DEVICES"] = "0"
42 + prefill_proc = Popen(prefill_cmd, env=prefill_env)
43 +
44 + # Start decode instance
45 + decode_cmd = [
46 + sys.executable,
47 + "-m",
48 + "vllm.entrypoints.openai.api_server",
49 + "--model",
50 + "meta-llama/Meta-Llama-3.1-8B-Instruct",
51 + "--port",
52 + "8200",
53 + "--gpu-memory-utilization",
54 + "0.5",
55 + "--max-model-len",
56 + "1000",
57 + "--kv-transfer-config",
58 + '{"kv_connector":"PyNcclConnector","kv_role":"kv_consumer",'\
59 + '"kv_rank":1,"kv_parallel_size":2}',
60 + ]
61 + decode_env = os.environ.copy()
62 + decode_env["CUDA_VISIBLE_DEVICES"] = "1"
63 + decode_proc = Popen(decode_cmd, env=decode_env)
64 +
65 + # Wait for servers to be ready
66 + assert wait_for_server(8100), "Prefill server did not start in time"
67 + assert wait_for_server(8200), "Decode server did not start in time"
68 +
69 + # Yield to the test function and handle teardown after tests
70 + yield
71 +
72 + # Cleanup: kill the processes
73 + prefill_proc.terminate()
74 + decode_proc.terminate()
75 +
76 + # Additional cleanup if needed
77 + prefill_proc.wait()
78 + decode_proc.wait()
79 +
80 +
81 +# Helper function to wait for server
82 +def wait_for_server(port, timeout=240):
83 + start_time = time.time()
84 + while time.time() - start_time < timeout:
85 + try:
86 + response = requests.get(f"http://localhost:{port}/v1/completions")
87 + if response.status_code in [200, 405]:
88 + return True
89 + except requests.ConnectionError:
90 + time.sleep(1)
91 + return False
92 +
93 +
94 +# Test function to send curl requests and validate responses
95 +@pytest.mark.parametrize("prompt", ["San Francisco is a", "Santa Clara is a"])
96 +def test_disaggregated_prefilling(prompt):
97 + # Send to prefill
98 + response = requests.post("http://localhost:8100/v1/completions",
99 + headers={"Content-Type": "application/json"},
100 + json={
101 + "model":
102 + "meta-llama/Meta-Llama-3.1-8B-Instruct",
103 + "prompt": prompt,
104 + "max_tokens": 1,
105 + "temperature": 0
106 + })
107 + assert response.status_code == 200
108 +
109 + # Send to decode
110 + response = requests.post("http://localhost:8200/v1/completions",
111 + headers={"Content-Type": "application/json"},
112 + json={
113 + "model":
114 + "meta-llama/Meta-Llama-3.1-8B-Instruct",
115 + "prompt": prompt,
116 + "max_tokens": 10,
117 + "temperature": 0
118 + })
119 + assert response.status_code == 200
tests/kv_transfer/module_test.py new
+64
@@ -0,0 +1,64 @@
1 +import subprocess
2 +import sys
3 +
4 +import pytest
5 +import torch
6 +
7 +
8 +def run_python_script(script_name, timeout):
9 + script_name = f'kv_transfer/{script_name}'
10 + try:
11 + # Start both processes asynchronously using Popen
12 + process0 = subprocess.Popen(
13 + [sys.executable, script_name],
14 + env={"RANK":
15 + "0"}, # Set the RANK environment variable for process 0
16 + stdout=sys.stdout, # Pipe stdout to current stdout
17 + stderr=sys.stderr, # Pipe stderr to current stderr
18 + )
19 +
20 + process1 = subprocess.Popen(
21 + [sys.executable, script_name],
22 + env={"RANK":
23 + "1"}, # Set the RANK environment variable for process 1
24 + stdout=sys.stdout, # Pipe stdout to current stdout
25 + stderr=sys.stderr, # Pipe stderr to current stderr
26 + )
27 +
28 + # Wait for both processes to complete, with a timeout
29 + process0.wait(timeout=timeout)
30 + process1.wait(timeout=timeout)
31 +
32 + # Check the return status of both processes
33 + if process0.returncode != 0:
34 + pytest.fail(
35 + f"Test {script_name} failed for RANK=0, {process0.returncode}")
36 + if process1.returncode != 0:
37 + pytest.fail(
38 + f"Test {script_name} failed for RANK=1, {process1.returncode}")
39 +
40 + except subprocess.TimeoutExpired:
41 + # If either process times out, terminate both and fail the test
42 + process0.terminate()
43 + process1.terminate()
44 + pytest.fail(f"Test {script_name} timed out")
45 + except Exception as e:
46 + pytest.fail(f"Test {script_name} failed with error: {str(e)}")
47 +
48 +
49 +# Define the test cases using pytest's parametrize
50 +@pytest.mark.parametrize(
51 + "script_name,timeout",
52 + [
53 + ("test_lookup_buffer.py",
54 + 60), # Second test case with a 60-second timeout
55 + ("test_send_recv.py", 120) # First test case with a 120-second timeout
56 + ])
57 +def test_run_python_script(script_name, timeout):
58 + # Check the number of GPUs
59 + if torch.cuda.device_count() < 2:
60 + pytest.skip(
61 + f"Skipping test {script_name} because <2 GPUs are available")
62 +
63 + # Run the test if there are at least 2 GPUs
64 + run_python_script(script_name, timeout)
tests/kv_transfer/test_lookup_buffer.py new
+160
@@ -0,0 +1,160 @@
1 +import os
2 +import random
3 +
4 +import torch
5 +from tqdm import tqdm
6 +
7 +from vllm.config import KVTransferConfig
8 +from vllm.distributed.kv_transfer.kv_lookup_buffer.simple_buffer import (
9 + SimpleBuffer)
10 +from vllm.distributed.kv_transfer.kv_pipe.pynccl_pipe import PyNcclPipe
11 +
12 +# TODO: the test depends on a lot of fields in the current implementation.
13 +# We should have standard interface instead direct field access
14 +
15 +
16 +def test_run(my_rank, buffer, device):
17 +
18 + # buffer should be empty in the beginning
19 + if my_rank == 0:
20 + assert buffer.buffer_size == 0
21 + assert len(buffer.buffer) == 0
22 +
23 + print("My rank: %d, device: %s" % (my_rank, device))
24 +
25 + # insert
26 + tokens = torch.tensor([1, 2, 3]).to(device)
27 + roi = (tokens > 0)
28 + if my_rank == 0:
29 + key = 2.0 * torch.ones([5, 6]).to(device)
30 + value = 3.0 * torch.ones([5, 6]).to(device)
31 +
32 + placeholder = torch.tensor([1]).to(device)
33 +
34 + buffer.insert(tokens, roi, key, value, placeholder)
35 +
36 + torch.distributed.barrier()
37 +
38 + # drop_select
39 + if my_rank == 1:
40 + tok, roi_, key, value, hidden = buffer.drop_select(tokens, roi)
41 + assert torch.allclose(tokens, tok)
42 + assert torch.allclose(roi, roi_)
43 + assert torch.allclose(key, 2.0 * torch.ones([5, 6], device=device))
44 + assert torch.allclose(value, 3.0 * torch.ones([5, 6], device=device))
45 + torch.distributed.barrier()
46 +
47 + if my_rank == 0:
48 + assert buffer.buffer_size == 0
49 + assert len(buffer.buffer) == 0
50 +
51 + print("Test run passed!")
52 +
53 +
54 +def stress_test(my_rank, buf, device):
55 +
56 + torch.distributed.barrier()
57 + torch.manual_seed(100)
58 +
59 + reqs = [
60 + (
61 + torch.rand(100).to(device), # tokens
62 + torch.ones(100).bool().to(device), # roi
63 + torch.rand(100).to(device), # key
64 + torch.rand(100).to(device), # value
65 + torch.rand(100).to(device), # hidden
66 + ) for i in tqdm(range(200))
67 + ]
68 +
69 + random.seed(my_rank)
70 + random.shuffle(reqs)
71 +
72 + torch.distributed.barrier()
73 +
74 + n = 0
75 +
76 + # the buffer size can only store 100 reqs
77 + # so the sender will occasionally block to wait for the receiver.
78 + for req in tqdm(reqs):
79 + if my_rank == 0:
80 + buf.insert(*req)
81 + else:
82 + tok, roi, k, v, h = req
83 + tok_, roi_, k_, v_, h_ = buf.drop_select(tok, roi)
84 +
85 + if tok_ is None:
86 + assert roi_ is None
87 + assert k_ is None
88 + assert v_ is None
89 + assert h_ is None
90 + n += 1
91 + else:
92 + assert torch.allclose(tok, tok_)
93 + assert torch.allclose(roi, roi_)
94 + assert torch.allclose(k, k_)
95 + assert torch.allclose(v, v_)
96 + assert torch.allclose(h, h_)
97 + print('Rank %d done' % my_rank)
98 + torch.distributed.barrier()
99 +
100 + if my_rank == 0:
101 + x = torch.tensor([0])
102 + torch.distributed.recv(x, 1)
103 + # the # of None received is the kv that are not selected
104 + assert x.item() == len(buf.buffer)
105 + # and the size of the buffer should be 2000 * buffer len
106 + print(buf.buffer_size)
107 + assert buf.buffer_size == 1700 * len(buf.buffer)
108 + else:
109 + torch.distributed.send(torch.tensor([n]), 0)
110 +
111 + print("Passed stress test!")
112 +
113 +
114 +if __name__ == "__main__":
115 +
116 + my_rank = int(os.environ['RANK'])
117 +
118 + torch.distributed.init_process_group(
119 + backend='gloo',
120 + init_method='tcp://localhost:12398',
121 + world_size=2,
122 + rank=my_rank,
123 + )
124 +
125 + print("initialized! My rank is %d" % my_rank)
126 +
127 + config = KVTransferConfig(
128 + kv_connector='PyNcclConnector',
129 + kv_buffer_device='cuda',
130 + kv_buffer_size=1e9,
131 + kv_rank=my_rank,
132 + kv_role="kv_both", # this arg doesn't matter in this test
133 + kv_parallel_size=2,
134 + kv_ip="127.0.0.1",
135 + kv_port=12345,
136 + )
137 +
138 + data_pipe = PyNcclPipe(
139 + local_rank=my_rank,
140 + config=config,
141 + device="cuda",
142 + port_offset=0,
143 + )
144 + cpu_pipe = PyNcclPipe(
145 + local_rank=my_rank,
146 + config=config,
147 + device="cpu",
148 + port_offset=1,
149 + )
150 +
151 + buffer = SimpleBuffer(cpu_pipe, data_pipe, 170000)
152 +
153 + test_run(my_rank, buffer, data_pipe.device)
154 +
155 + stress_test(my_rank, buffer, data_pipe.device)
156 +
157 + buffer.close()
158 + data_pipe.close()
159 + cpu_pipe.close()
160 + print('Done')
tests/kv_transfer/test_lookup_buffer.sh new
+3
@@ -0,0 +1,3 @@
1 +#!/bin/bash
2 +RANK=0 python test_lookup_buffer.py &
3 +RANK=1 python test_lookup_buffer.py &
\ No newline at end of file
tests/kv_transfer/test_send_recv.py new
+155
@@ -0,0 +1,155 @@
1 +import os
2 +import time
3 +from typing import List
4 +
5 +import torch
6 +from tqdm import tqdm
7 +
8 +from vllm.config import KVTransferConfig
9 +from vllm.distributed.kv_transfer.kv_pipe.pynccl_pipe import PyNcclPipe
10 +
11 +
12 +def test_run(my_rank, pipe):
13 + # test run
14 + x = torch.tensor([1]).to(pipe.device)
15 + y = torch.tensor([[2., 3., 4., 8.]]).to(pipe.device)
16 + if my_rank == 0:
17 + pipe.send_tensor(x)
18 + print("sent tensor x")
19 + pipe.send_tensor(y)
20 + print("sent tensor y")
21 + x2 = pipe.recv_tensor()
22 + print("received x2 = ", x2)
23 + y2 = pipe.recv_tensor()
24 + print("received y2 = ", x2)
25 +
26 + else:
27 + x2 = pipe.recv_tensor()
28 + print("received x2 = ", x2)
29 + y2 = pipe.recv_tensor()
30 + print("received y2 = ", x2)
31 + pipe.send_tensor(x)
32 + print("sent tensor x")
33 + pipe.send_tensor(y)
34 + print("sent tensor y")
35 +
36 + assert torch.allclose(x, x2)
37 + assert torch.allclose(y, y2)
38 +
39 +
40 +def stress_test(my_rank, pipe):
41 +
42 + torch.distributed.barrier()
43 +
44 + tensors: List[torch.Tensor] = []
45 +
46 + torch.manual_seed(0)
47 +
48 + for i in tqdm(range(500)):
49 + mean = torch.rand(1).item() * 100
50 + std = torch.rand(1).item() * 100
51 + size = torch.randint(900, 1000, (2, ))
52 + x = torch.normal(mean * 1.0, std * 1.0,
53 + size=size.tolist()).to(pipe.device)
54 +
55 + # 5% probability of sending a None
56 + if torch.rand(1).item() < 0.05:
57 + tensors.append(None)
58 + tensors.append(None)
59 + tensors.append(None)
60 + else:
61 + tensors.append(x)
62 + tensors.append(x.mean().unsqueeze(0))
63 + tensors.append(x.std().unsqueeze(0))
64 +
65 + torch.distributed.barrier()
66 +
67 + for i in tqdm(range(500)):
68 + if my_rank == int((i % 10) > 3):
69 + pipe.send_tensor(tensors[3 * i])
70 + pipe.send_tensor(tensors[3 * i + 1])
71 + pipe.send_tensor(tensors[3 * i + 2])
72 + else:
73 + x = pipe.recv_tensor()
74 + mean = pipe.recv_tensor()
75 + std = pipe.recv_tensor()
76 +
77 + if x is None:
78 + assert mean is None
79 + assert std is None
80 + else:
81 + assert torch.allclose(x, tensors[3 * i])
82 + assert x.mean() == mean[0]
83 + assert x.std() == std[0]
84 +
85 + torch.distributed.barrier()
86 +
87 +
88 +def latency_test(my_rank, pipe, nelement, ntensor):
89 +
90 + latencies = []
91 +
92 + torch.distributed.barrier()
93 +
94 + for i in tqdm(range(500)):
95 +
96 + tensors = []
97 +
98 + if my_rank == 0:
99 + # create tensor
100 + tensors = [
101 + torch.rand(nelement).to(pipe.device) for _ in range(ntensor)
102 + ]
103 +
104 + torch.distributed.barrier()
105 +
106 + if my_rank == 0:
107 + t = torch.tensor([time.time()],
108 + dtype=torch.float64).to(pipe.device)
109 + for tensor in tensors:
110 + pipe.send_tensor(tensor)
111 + pipe.send_tensor(t)
112 + else:
113 + for _ in range(ntensor):
114 + pipe.recv_tensor()
115 + t = pipe.recv_tensor()
116 + latencies.append(time.time() - t.item())
117 +
118 + torch.distributed.barrier()
119 +
120 + print('Latency test passed.')
121 + print('Latency:', torch.tensor(latencies).mean().item() * 1000, 'ms')
122 +
123 +
124 +if __name__ == "__main__":
125 +
126 + my_rank = int(os.environ['RANK'])
127 +
128 + torch.distributed.init_process_group(
129 + backend='gloo',
130 + init_method='tcp://localhost:12398',
131 + world_size=2,
132 + rank=my_rank,
133 + )
134 +
135 + config = KVTransferConfig(
136 + kv_connector='PyNcclConnector',
137 + kv_buffer_device='cuda',
138 + kv_buffer_size=1e9,
139 + kv_rank=my_rank,
140 + kv_role="kv_both", # this arg doesn't matter in this test
141 + kv_parallel_size=2,
142 + kv_ip="127.0.0.1",
143 + kv_port=12345,
144 + )
145 +
146 + pipe = PyNcclPipe(
147 + local_rank=my_rank,
148 + config=config,
149 + )
150 +
151 + test_run(my_rank, pipe)
152 + stress_test(my_rank, pipe)
153 +
154 + # Use this function if you want to test the latency of pipe impl.
155 + # latency_test(my_rank, pipe, 1024 * 8 * 128, 80)
tests/kv_transfer/test_send_recv.sh new
+3
@@ -0,0 +1,3 @@
1 +#!/bin/bash
2 +RANK=0 python3 test_send_recv.py &
3 +RANK=1 python3 test_send_recv.py &
\ No newline at end of file
vllm/config.py
+84
@@ -2052,6 +2052,88 @@ class ObservabilityConfig:
2052 f"installed. Original error:\n{otel_import_error_traceback}")
2053
2054
2055 +class KVTransferConfig(BaseModel):
2056 + """Configuration for distributed KV cache transfer."""
2057 +
2058 + # The KV connector for vLLM to transmit KV caches between vLLM instances.
2059 + kv_connector: Optional[str] = None
2060 +
2061 + # The device used by kv connector to buffer the KV cache.
2062 + # Currently only support 'cuda'.
2063 + kv_buffer_device: Optional[str] = "cuda"
2064 +
2065 + # The buffer size for TorchDistributedConnector. Measured in number of
2066 + # bytes. Recommended value: 1e9 (about 1GB).
2067 + kv_buffer_size: float = 1e9
2068 +
2069 + # Whether this vLLM instance produces, consumes KV cache, or both. Choices
2070 + # are 'kv_producer', 'kv_consumer', and 'both'.
2071 + kv_role: Optional[str] = None
2072 +
2073 + # The rank of this vLLM instance in the KV cache transfer. Typical value:
2074 + # 0 for prefill instance, 1 for decode instance.
2075 + # Currently only 1P1D is supported.
2076 + kv_rank: Optional[int] = None
2077 +
2078 + # The number of parallel instances for KV cache transfer. For
2079 + # PyNcclConnector, this should be 2.
2080 + kv_parallel_size: int = 1
2081 +
2082 + # The KV connector ip, used to build distributed connection
2083 + kv_ip: str = "127.0.0.1"
2084 +
2085 + # The KV connector port, used to build distributed connection
2086 + kv_port: int = 14579
2087 +
2088 + @classmethod
2089 + def from_cli(cls, cli_value: str) -> "KVTransferConfig":
2090 + """Parse the CLI value for the compilation config."""
2091 + return KVTransferConfig.model_validate_json(cli_value)
2092 +
2093 + def model_post_init(self, __context: Any) -> None:
2094 + if all([
2095 + self.kv_connector is not None,
2096 + self.kv_connector != "PyNcclConnector"
2097 + ]):
2098 + raise ValueError(f"Unsupported kv_connector: {self.kv_connector}. "
2099 + f"Supported connectors are "
2100 + f"`PyNcclConnector`.")
2101 +
2102 + if self.kv_role is not None and self.kv_role not in [
2103 + "kv_producer", "kv_consumer", "kv_both"
2104 + ]:
2105 + raise ValueError(
2106 + f"Unsupported kv_role: {self.kv_role}. "
2107 + f"Supported roles are `kv_producer`, `kv_consumer`, "
2108 + f"and `kv_both`")
2109 +
2110 + if self.kv_connector is not None and self.kv_role is None:
2111 + raise ValueError("Please specify kv_disagg_role when kv_connector "
2112 + "is set, supported roles are `kv_producer`, "
2113 + "`kv_consumer`, and `kv_both`")
2114 +
2115 + @property
2116 + def is_kv_transfer_instance(self) -> bool:
2117 + return self.kv_connector is not None and \
2118 + self.kv_role in ["kv_producer", "kv_consumer", "kv_both"]
2119 +
2120 + @property
2121 + def need_kv_parallel_group(self) -> bool:
2122 + # for those database-based connector, vLLM does not need to create
2123 + # parallel group, and in that case the kv parallel size will be 1.
2124 + return self.kv_connector is not None and self.kv_parallel_size > 1
2125 +
2126 + @property
2127 + def is_kv_producer(self) -> bool:
2128 + return self.kv_connector is not None and \
2129 + self.kv_role in ["kv_producer", "kv_both"]
2130 +
2131 + @property
2132 + def is_kv_consumer(self) -> bool:
2133 + return self.kv_connector is not None and \
2134 + self.kv_role in ["kv_consumer", "kv_both"]
2135 +
2136 +
2137 class CompilationLevel:
2138 # constants for the levels of the compilation process
2139 NO_COMPILATION = 0
@@ -2317,6 +2399,8 @@ class VllmConfig:
2399 quant_config: Optional[QuantizationConfig] = None
2400 compilation_config: CompilationConfig = field(default=None,
2401 init=True) # type: ignore
2402 + kv_transfer_config: KVTransferConfig = field(default=None,
2403 + init=True) # type: ignore
2404
2405 @staticmethod
2406 def _get_quantization_config(
vllm/distributed/kv_transfer/README.md new
+30
@@ -0,0 +1,30 @@
1 +
2 +# Distributed KV cache transfer
3 +
4 +This folder implements distributed KV cache transfer across vLLM instances.
5 +Currently the main usecase is for disaggregated prefilling.
6 +
7 +## Abstractions
8 +
9 +The KV cache transfer contains three layer of abstractions:
10 +
11 +- KV pipe: a FIFO pipe for torch.tensor transmission. Key APIs: `send_tensor` and `recv_tensor`.
12 +- KV lookup buffer: a lookup buffer for KV caches. Key: the tokens, value: the KV caches (and/or hidden states). Key APIs: `insert` and `drop_select` (similar to SQL semantics).
13 +- KV connector: a connector that connects the KV pipe and KV lookup buffer to vLLM. Key APIs: `send_kv_caches_and_hidden_states` and `recv_kv_caches_and_hidden_states`.
14 +
15 +Why we need KV lookup buffer: FIFO pipe itself is not enough as prefill vLLM worker may process requests in a different order compared to decode vLLM worker. Say the QPS is really high, prefill worker may handle requests in order A -> B -> C, but the decode worker may process request C first. This is not the case that can be naturally handled by FIFO pipe, so we provide KV lookup buffer to help translate a FIFO pipe to a lookup buffer.
16 +
17 +NOTE: KV pipe layer is bypassible: you can skip this layer if your distributed
18 +communication service already supports key-value-based lookup (like redis or
19 +RDMA database).
20 +
21 +NOTE: If you want to not only transfer KV caches, but adjust the model execution flow of vLLM as well (for example, allow vLLM to receive KV caches on some tokens and do prefill on the remaining tokens), you can bypass both KV pipe layer and KV lookup buffer layer, and directly implement on KV connector layer. Bear in mind that as vLLM's model input is constantly changing, this implementation will likely be broken when vLLM has new updates.
22 +
23 +## Disaggregated prefilling
24 +
25 +The example usage is in [this file](../../../examples/disaggregated_prefill.sh).
26 +
27 +Here is the diagram of how we run disaggretgated prefilling.
28 +
29 +![Disaggregated prefill workflow](./disagg_prefill_workflow.jpg)
30 +
vllm/distributed/kv_transfer/__init__.py
vllm/distributed/kv_transfer/disagg_prefill_workflow.jpg
Binary files /dev/null and b/vllm/distributed/kv_transfer/disagg_prefill_workflow.jpg differ
vllm/distributed/kv_transfer/kv_connector/__init__.py
vllm/distributed/kv_transfer/kv_connector/base.py new
+122
@@ -0,0 +1,122 @@
1 +"""
2 +KVConnectorBase Class for Distributed KV Cache & Hidden State communication
3 +
4 +The class provides two primary abstract methods:
5 +1. send_kv_caches_and_hidden_states(): Send KV caches and hidden states
6 +2. recv_kv_caches_and_hidden_states(): Recv KV caches and hidden states
7 +"""
8 +
9 +from abc import ABC, abstractmethod
10 +from typing import TYPE_CHECKING, List, Tuple, Union
11 +
12 +import torch
13 +
14 +from vllm.sequence import IntermediateTensors
15 +
16 +if TYPE_CHECKING:
17 + from vllm.config import VllmConfig
18 + from vllm.worker.model_runner import ModelInputForGPUWithSamplingMetadata
19 +
20 +
21 +class KVConnectorBase(ABC):
22 + """
23 + Abstract base class for a KV connector.
24 +
25 + The class provides two primary abstract methods:
26 + 1. send_kv_caches_and_hidden_states(): Send KV caches and hidden states
27 + 2. recv_kv_caches_and_hidden_states(): Recv KV caches and hidden states
28 + """
29 +
30 + @abstractmethod
31 + def __init__(
32 + self,
33 + rank: int,
34 + local_rank: int,
35 + config: "VllmConfig",
36 + ):
37 + raise NotImplementedError
38 +
39 + @abstractmethod
40 + def close(self) -> None:
41 + """Close the buffer and release resources.
42 +
43 + This method is responsible for cleaning up resources related to the
44 + connector when it is no longer needed.
45 +
46 + Raises:
47 + NotImplementedError: This method must be implemented in subclasses.
48 + """
49 + raise NotImplementedError
50 +
51 + @abstractmethod
52 + def send_kv_caches_and_hidden_states(
53 + self,
54 + model_executable: torch.nn.Module,
55 + model_input: "ModelInputForGPUWithSamplingMetadata",
56 + kv_caches: List[torch.Tensor],
57 + hidden_or_intermediate_states: Union[torch.Tensor,
58 + IntermediateTensors],
59 + ) -> None:
60 + """
61 + Send KV caches and hidden states to the connector.
62 +
63 + This method processes the input tokens, KV caches, and
64 + hidden/intermediate states for a given model and sends the data to the
65 + decode instance.
66 +
67 + Args:
68 + model_executable (torch.nn.Module): The model executable containing
69 + start and end layer information.
70 + model_input (ModelInputForGPUWithSamplingMetadata): The input
71 + metadata from vLLM.
72 + kv_caches (List[torch.Tensor]): List of KV caches (keys and values)
73 + for each layer.
74 + hidden_or_intermediate_states (Union[torch.Tensor,
75 + IntermediateTensors]):
76 + The hidden or intermediate states associated with the tokens.
77 +
78 + Returns:
79 + None
80 +
81 + """
82 +
83 + raise NotImplementedError
84 +
85 + @abstractmethod
86 + def recv_kv_caches_and_hidden_states(
87 + self, model_executable: torch.nn.Module,
88 + model_input: "ModelInputForGPUWithSamplingMetadata",
89 + kv_caches: List[torch.Tensor]
90 + ) -> Tuple[Union[torch.Tensor, IntermediateTensors], bool,
91 + "ModelInputForGPUWithSamplingMetadata"]:
92 + """
93 + Receive KV caches and hidden states from the connector.
94 +
95 + This method attempts to retrieve KV caches and hidden states for input
96 + tokens. If all required KV caches and hidden states are received, it
97 + will bypass model input, else it will fall back to normal vLLM model
98 + forwarding.
99 +
100 + Args:
101 + model_executable (torch.nn.Module):
102 + The model executable from vLLM modelrunner.
103 + model_input (ModelInputForGPUWithSamplingMetadata):
104 + The model input from vLLM modelrunner.
105 + kv_caches (List[torch.Tensor]):
106 + List of KV caches for each layer.
107 +
108 + Returns:
109 + - hidden_or_intermediate_states (torch.Tensor or
110 + IntermediateTensors):
111 + Concatenated hidden states if all required data is retrieved,
112 + otherwise `None`.
113 + - bypass_model_exec (bool):
114 + Indicates whether the model execution can be skipped (True) or
115 + needs to be redone (False).
116 + - model_input (ModelInputForGPUWithSamplingMetadata):
117 + Optionally adjusted input metadata for re-execution when
118 + `bypass_model_exec=False`.
119 +
120 + """
121 +
122 + raise NotImplementedError
vllm/distributed/kv_transfer/kv_connector/factory.py new
+19
@@ -0,0 +1,19 @@
1 +from typing import TYPE_CHECKING
2 +
3 +from .base import KVConnectorBase
4 +
5 +if TYPE_CHECKING:
6 + from vllm.config import VllmConfig
7 +
8 +
9 +class KVConnectorFactory:
10 +
11 + @staticmethod
12 + def create_connector(rank: int, local_rank: int,
13 + config: "VllmConfig") -> KVConnectorBase:
14 + if config.kv_transfer_config.kv_connector == 'PyNcclConnector':
15 + from .simple_connector import SimpleConnector
16 + return SimpleConnector(rank, local_rank, config)
17 + else:
18 + raise ValueError(f"Unsupported connector type: "
19 + f"{config.kv_connector}")
vllm/distributed/kv_transfer/kv_connector/simple_connector.py new
+261
@@ -0,0 +1,261 @@
1 +"""
2 +Simple KV Cache Connector for Distributed Machine Learning Inference
3 +
4 +The SimpleConnector transfers KV caches between prefill vLLM worker (KV cache
5 +producer) and decode vLLM worker (KV cache consumer) using PyNcclPipe.
6 +
7 +But the logic can be extended to support other pipe and lookup buffer.
8 +"""
9 +from typing import TYPE_CHECKING, List, Optional, Tuple, Union
10 +
11 +import torch
12 +
13 +from vllm import _custom_ops as ops
14 +from vllm.config import VllmConfig
15 +from vllm.distributed.kv_transfer.kv_connector.base import KVConnectorBase
16 +from vllm.distributed.kv_transfer.kv_lookup_buffer.simple_buffer import (
17 + SimpleBuffer)
18 +from vllm.distributed.kv_transfer.kv_pipe.pynccl_pipe import PyNcclPipe
19 +from vllm.logger import init_logger
20 +from vllm.sequence import IntermediateTensors
21 +
22 +if TYPE_CHECKING:
23 + from vllm.worker.model_runner import ModelInputForGPUWithSamplingMetadata
24 +
25 +logger = init_logger(__name__)
26 +
27 +
28 +class SimpleConnector(KVConnectorBase):
29 +
30 + def __init__(
31 + self,
32 + rank: int,
33 + local_rank: int,
34 + config: VllmConfig,
35 + ):
36 +
37 + self.config = config.kv_transfer_config
38 +
39 + logger.info("Initializing PyNcclConfig under kv_transfer_config %s",
40 + self.config)
41 +
42 + self.lookup_buffer_size = self.config.kv_buffer_size
43 +
44 + self.producer_buffer: Optional[SimpleBuffer] = None
45 + self.consumer_buffer: Optional[SimpleBuffer] = None
46 +
47 + # 2 pipes for every rank in the world
48 + port_offset_base = 2 * rank
49 +
50 + # In disaggregated prefill, the prefill vLLM only uses send pipe
51 + # and the decode vLLM only uses recv pipe
52 + if self.config.is_kv_producer:
53 +
54 + self.producer_data_pipe = PyNcclPipe(
55 + local_rank=local_rank,
56 + config=self.config,
57 + port_offset=port_offset_base,
58 + )
59 + self.producer_signal_pipe = PyNcclPipe(
60 + local_rank=local_rank,
61 + config=self.config,
62 + port_offset=port_offset_base + 1,
63 + device="cpu",
64 + )
65 + self.producer_buffer = SimpleBuffer(self.producer_signal_pipe,
66 + self.producer_data_pipe,
67 + self.config.kv_buffer_size)
68 +
69 + else:
70 +
71 + # the current vLLM instance is KV consumer, so it needs to connect
72 + # its recv pipe to the send pipe of KV producder
73 + self.consumer_data_pipe = PyNcclPipe(
74 + local_rank=local_rank,
75 + config=self.config,
76 + port_offset=port_offset_base,
77 + )
78 + self.consumer_signal_pipe = PyNcclPipe(
79 + local_rank=local_rank,
80 + config=self.config,
81 + port_offset=port_offset_base + 1,
82 + device="cpu",
83 + )
84 + self.consumer_buffer = SimpleBuffer(
85 + self.consumer_signal_pipe,
86 + self.consumer_data_pipe,
87 + self.config.kv_buffer_size,
88 + )
89 +
90 + def select(self, input_tokens: Optional[torch.Tensor],
91 + roi: Optional[torch.Tensor]) -> List[Optional[torch.Tensor]]:
92 +
93 + assert self.consumer_buffer is not None, "Please initialize the "\
94 + "consumer buffer before calling select."
95 + return self.consumer_buffer.drop_select(input_tokens, roi)
96 +
97 + def insert(self, input_tokens: torch.Tensor, roi: torch.Tensor,
98 + key: torch.Tensor, value: torch.Tensor,
99 + hidden: torch.Tensor) -> None:
100 +
101 + assert self.producer_buffer is not None, "Please initialize the "\
102 + "producer buffer before calling insert."
103 +
104 + self.producer_buffer.insert(input_tokens, roi, key, value, hidden)
105 +
106 + def send_kv_caches_and_hidden_states(
107 + self,
108 + model_executable: torch.nn.Module,
109 + model_input: "ModelInputForGPUWithSamplingMetadata",
110 + kv_caches: List[torch.Tensor],
111 + hidden_or_intermediate_states: Union[torch.Tensor,
112 + IntermediateTensors],
113 + ) -> None:
114 +
115 + input_tokens_tensor = model_input.input_tokens
116 + seq_lens = model_input.attn_metadata.seq_lens
117 + slot_mapping_flat = model_input.attn_metadata.slot_mapping.flatten()
118 + start_layer = model_executable.model.start_layer
119 + end_layer = model_executable.model.end_layer
120 +
121 + # query_lens contains new KV caches that are added to vLLM.
122 + # so we will send them to decode instance
123 + # FIXME(Kuntai): This assume that all requests are prefill.
124 + for idx, slen in enumerate(seq_lens):
125 + start_pos = sum(seq_lens[:idx])
126 + end_pos = start_pos + slen
127 + current_tokens = input_tokens_tensor[start_pos:end_pos]
128 +
129 + keys, values = [], []
130 +
131 + for layer_id in range(start_layer, end_layer):
132 + kv_cache = kv_caches[layer_id - start_layer]
133 +
134 + _, _, num_heads, head_size = kv_cache[0].shape
135 +
136 + key_cache = kv_cache[0].reshape(-1, num_heads, head_size)
137 + value_cache = kv_cache[1].reshape(-1, num_heads, head_size)
138 +
139 + current_slot_mapping = slot_mapping_flat[start_pos:end_pos]
140 +
141 + keys.append(key_cache[current_slot_mapping].unsqueeze(0))
142 + values.append(value_cache[current_slot_mapping].unsqueeze(0))
143 +
144 + keys = torch.cat(keys, dim=0)
145 + values = torch.cat(values, dim=0)
146 +
147 + self.insert(current_tokens,
148 + torch.ones_like(current_tokens,
149 + dtype=bool), keys, values,
150 + hidden_or_intermediate_states[start_pos:end_pos])
151 +
152 + logger.debug("[rank%d]: KV send DONE.", torch.distributed.get_rank())
153 +
154 + def recv_kv_caches_and_hidden_states(
155 + self, model_executable: torch.nn.Module,
156 + model_input: "ModelInputForGPUWithSamplingMetadata",
157 + kv_caches: List[torch.Tensor]
158 + ) -> Tuple[Union[torch.Tensor, IntermediateTensors], bool,
159 + "ModelInputForGPUWithSamplingMetadata"]:
160 +
161 + # When bypass_model_exec is set to False, it means that at least for one
162 + # request its corresponding KV cache or hidden state is missing.
163 + # In this case we need to do prefilling to recompute missing KV cache
164 + # and hidden states.
165 + bypass_model_exec = True
166 +
167 + input_tokens_tensor = model_input.input_tokens
168 + seq_lens = model_input.attn_metadata.seq_lens
169 + slot_mapping = model_input.attn_metadata.slot_mapping.flatten()
170 +
171 + hidden_or_intermediate_states_for_one_req = []
172 +
173 + input_tokens_list = []
174 + num_computed_tokens_list = []
175 + start_pos_list = []
176 +
177 + # enumerate different requests
178 + # FIXME(Kuntai): This impl assumes that all requests are prefill.
179 + for idx, slen in enumerate(seq_lens):
180 +
181 + start_pos = sum(seq_lens[:idx])
182 + end_pos = start_pos + slen
183 + current_tokens = input_tokens_tensor[start_pos:end_pos]
184 + num_tokens = slen
185 +
186 + # collecting data for rebuilding the input
187 + input_tokens_list.append(current_tokens)
188 + start_pos_list.append(start_pos)
189 +
190 + ret = self.select(current_tokens,
191 + torch.ones_like(current_tokens, dtype=bool))
192 + if ret[0] is None:
193 + # didn't find any match.
194 + bypass_model_exec = False
195 + num_computed_tokens_list.append(0)
196 + continue
197 +
198 + roi: torch.Tensor = ret[1]
199 + keys: torch.Tensor = ret[2]
200 + values: torch.Tensor = ret[3]
201 + hidden: torch.Tensor = ret[4]
202 +
203 + num_computed_tokens = roi.shape[0]
204 + num_computed_tokens_list.append(num_computed_tokens)
205 +
206 + # check if both KV cache and the hidden states are received
207 + # If not, need to redo the forwarding to compute missing states
208 + if not all([(num_computed_tokens == num_tokens), hidden is not None
209 + ]):
210 + bypass_model_exec = False
211 +
212 + # update the end position based on how many tokens are cached.
213 + end_pos = start_pos + num_computed_tokens
214 +
215 + # put received KV caches into paged memory
216 + for i in range(model_executable.model.start_layer,
217 + model_executable.model.end_layer):
218 +
219 + kv_cache = kv_caches[i - model_executable.model.start_layer]
220 + layer = model_executable.model.layers[i]
221 +
222 + key_cache, value_cache = kv_cache[0], kv_cache[1]
223 + ops.reshape_and_cache_flash(
224 + keys[i - model_executable.model.start_layer].to(
225 + key_cache.device),
226 + values[i - model_executable.model.start_layer].to(
227 + value_cache.device),
228 + key_cache,
229 + value_cache,
230 + slot_mapping[start_pos:end_pos],
231 + layer.self_attn.attn.kv_cache_dtype,
232 + layer.self_attn.attn._k_scale,
233 + layer.self_attn.attn._v_scale,
234 + )
235 +
236 + hidden_or_intermediate_states_for_one_req.append(hidden)
237 +
238 + if not bypass_model_exec:
239 + # Some of the KV cache is not retrieved
240 + # Here we will fall back to normal model forwarding
241 + # But optionally you can adjust model_input so that you only do
242 + # prefilling on those tokens that are missing KV caches.
243 + logger.debug(
244 + "[rank%d]: Failed to receive all KVs and hidden "
245 + "states, redo model forwarding.", torch.distributed.get_rank())
246 + hidden_or_intermediate_states = None
247 +
248 + else:
249 + logger.debug(
250 + "[rank%d]: Successfully received all KVs and hidden "
251 + "states, skip model forwarding.", torch.distributed.get_rank())
252 + hidden_or_intermediate_states = torch.cat(
253 + hidden_or_intermediate_states_for_one_req, dim=0)
254 +
255 + return hidden_or_intermediate_states, bypass_model_exec, model_input
256 +
257 + def close(self):
258 + self.producer_data_pipe.close()
259 + self.producer_signal_pipe.close()
260 + self.consumer_data_pipe.close()
261 + self.consumer_signal_pipe.close()
vllm/distributed/kv_transfer/kv_lookup_buffer/__init__.py
vllm/distributed/kv_transfer/kv_lookup_buffer/base.py new
+108
@@ -0,0 +1,108 @@
1 +"""
2 +This file contains a new class `KVLookupBufferBase` that allows developers to
3 +think of KV cache operations as inserting new KV cache entries (`insert`)
4 +into the lookup buffer and querying existing KV caches (`drop_select`)
5 +from the lookup buffer.
6 +
7 +All distributed communications are abstracted behind this class.
8 +"""
9 +
10 +from abc import ABC, abstractmethod
11 +from typing import List, Optional
12 +
13 +import torch
14 +
15 +
16 +class KVLookupBufferBase(ABC):
17 + """
18 + Abstract base class for a lookup buffer.
19 +
20 + This class provides an abstraction for a key-value (KV) cache lookup buffer.
21 +
22 + The key of the lookup buffer:
23 + - input_tokens: token IDs of the request
24 + - roi: a binary mask on top of input_tokens.
25 + - Purpose of roi: Since KV cache may only be available for a subset of
26 + tokens in the input (for example, when vLLM is connected to an external
27 + KV cache service), roi specifies the subset of tokens that the KV cache
28 + is associated with.
29 + - NOTE: roi can be further extended to describe which part of KV the
30 + current process is holding (each process may only hold a part of KV
31 + due to TP and PP). This is not implemented for now.
32 +
33 + The value of the lookup buffer:
34 + - key: the key tensor in the KV cache
35 + - value: the value tensor in the KV cache
36 + - hidden: the final hidden state generated by model forwarding. This allows
37 + vLLM to bypass further model forwarding by transmitting the hidden state.
38 + """
39 +
40 + @abstractmethod
41 + def insert(self, input_tokens: torch.Tensor, roi: torch.Tensor,
42 + key: torch.Tensor, value: torch.Tensor,
43 + hidden: torch.Tensor) -> None:
44 + """Insert into the lookup buffer.
45 +
46 + The functionality is similar to the following python statement
47 + ```
48 + buffer[input_tokens, roi] = [key, value, hidden]
49 + ```
50 +
51 + FIXME: in the future, we should only have two arguments, key and value,
52 + where key is a tensor dict and value is a tensor dict.
53 +
54 + FIXME: we should transmit both sampler outputs and the hidden states.
55 +
56 + Args:
57 + input_tokens (torch.Tensor): token IDs.
58 + roi (torch.Tensor): A binary mask on top of the input tokens
59 + key (torch.Tensor): The key tensor in the KV cache.
60 + value (torch.Tensor): The value tensor in the KV cache.
61 + hidden (torch.Tensor): The final hidden state tensor generated
62 + during model forwarding to bypass model
63 + forwarding.
64 +
65 + Raises:
66 + NotImplementedError: This method must be implemented in subclasses.
67 + """
68 + raise NotImplementedError
69 +
70 + @abstractmethod
71 + def drop_select(
72 + self, input_tokens: Optional[torch.Tensor],
73 + roi: Optional[torch.Tensor]) -> List[Optional[torch.Tensor]]:
74 + """Select and *drop* KV cache entries from the lookup buffer.
75 +
76 + The functionality is similar to the following python statements
77 + ```
78 + ret = buffer.pop(input_tokens, roi)
79 + return ret
80 + ```
81 +
82 + If `input_tokens` and `roi` is `None`, it means selecting any of the
83 + KV caches in the buffer, return, and remove it from the buffer, useful
84 + when offloading KV cache to KV cache storage service.
85 +
86 + Args:
87 + input_tokens (torch.Tensor): token IDs.
88 + roi (torch.Tensor): A binary mask on top of the input tokens
89 +
90 + Returns:
91 + List[Optional[torch.Tensor]]: A list of tensors. Can be None.
92 +
93 + Raises:
94 + NotImplementedError: This method must be implemented in subclasses.
95 + """
96 + raise NotImplementedError
97 +
98 + @abstractmethod
99 + def close(self) -> None:
100 + """Close the buffer and release resources.
101 +
102 + This method is responsible for cleaning up resources related to the
103 + lookup buffer when it is no longer needed.
104 +
105 + Raises:
106 + NotImplementedError: This method must be implemented in subclasses.
107 + """
108 + raise NotImplementedError
vllm/distributed/kv_transfer/kv_lookup_buffer/simple_buffer.py new
+242
@@ -0,0 +1,242 @@
1 +"""
2 + Implements a distributed key-value (KV) cache transfer mechanism.
3 +
4 + Key Features:
5 + - Distributed KV cache transmission using PyNccl pipes.
6 + - Non-blocking `insert`, blocking `drop_select`.
7 + - Use CPU signal pipe to avoid racing condition
8 + - Handles buffer size constraints and provide backpressure mechanism to
9 + stop the prefill instance when the decode instance is slow.
10 +"""
11 +import threading
12 +import time
13 +from collections import deque
14 +from typing import Deque, List, Optional, Union
15 +
16 +import torch
17 +
18 +from vllm.distributed.kv_transfer.kv_lookup_buffer.base import (
19 + KVLookupBufferBase)
20 +from vllm.distributed.kv_transfer.kv_pipe.base import KVPipeBase
21 +from vllm.logger import init_logger
22 +
23 +logger = init_logger(__name__)
24 +
25 +
26 +class SimpleBuffer(KVLookupBufferBase):
27 +
28 + def __init__(self, signal_pipe: KVPipeBase, data_pipe: KVPipeBase,
29 + buffer_size_thresh: float):
30 + """
31 + signal_pipe: on CPU
32 +
33 + NOTE: on-device recv will block all threads in the process, making the
34 + KV cache producer unable to listen to new request while transmitting
35 + KV cache. Luckily CPU recv only blocks the current thread so we use
36 + CPU recv to listen to new request.
37 +
38 + data_pipe: on device (e.g. GPU)
39 + """
40 +
41 + self.buffer: Deque[List[torch.Tensor]] = deque()
42 +
43 + self.buffer_size = 0
44 + self.buffer_size_threshold = buffer_size_thresh
45 + self.buffer_lock = threading.Lock()
46 + self.signal_pipe = signal_pipe
47 + self.data_pipe = data_pipe
48 + self.request_handling_thread: Optional[threading.Thread] = None
49 +
50 + self.normal_signal = torch.tensor([0], device="cpu")
51 + self.end_signal = None
52 +
53 + def _matches(self, tokens_roi_sender: List[torch.Tensor],
54 + tokens_roi_recver: List[torch.Tensor]):
55 +
56 + # tokens_roi_sender: tokens and roi of the producer (in the buffer)
57 + # tokens_roi_recver: tokens and roi of the consumer (query)
58 +
59 + tokens_sender = tokens_roi_sender[0]
60 + tokens_recver = tokens_roi_recver[0]
61 + roi_sender = tokens_roi_sender[1]
62 + roi_recver = tokens_roi_recver[1]
63 +
64 + if tokens_recver is None:
65 + # consumer sends an empty request
66 + # semantics: DROP SELECT * LIMIT 1
67 + # so any of the data in the buffer can be drop-selected
68 + return True
69 +
70 + # Assuming that roi is a binary mask on tokens
71 + tokens_sender = tokens_sender[roi_sender]
72 + tokens_recver = tokens_recver[roi_recver]
73 +
74 + # simple common prefix matching
75 + min_length = min(len(tokens_sender), len(tokens_recver))
76 + if torch.allclose(tokens_sender[:min_length],
77 + tokens_recver[:min_length]):
78 + return min_length
79 +
80 + return 0
81 +
82 + def _send_tensor_and_dec_size(self,
83 + tensor: Optional[torch.Tensor]) -> None:
84 +
85 + assert tensor is not None, "Use self.data_pipe.send(None) instead"
86 + self.buffer_size -= tensor.element_size() * tensor.numel()
87 + if tensor.dtype == torch.bool:
88 + tensor = tensor.float()
89 + self.data_pipe.send_tensor(tensor)
90 +
91 + def _get_element_size(self, data: Optional[Union[List, torch.Tensor]]):
92 +
93 + if isinstance(data, torch.Tensor):
94 + return data.element_size() * data.numel()
95 + if not data:
96 + # cannot perform `not data` on a tensor
97 + # so this check needs to go after the check above
98 + return 0
99 +
100 + raise AssertionError(f"Unknown data type {type(data)}")
101 +
102 + def _add_to_buffer(self, input_tokens: torch.Tensor, roi: torch.Tensor,
103 + key: torch.Tensor, value: torch.Tensor,
104 + hidden: torch.Tensor):
105 +
106 + if isinstance(input_tokens, torch.Tensor):
107 + input_tokens = input_tokens.clone()
108 + if isinstance(roi, torch.Tensor):
109 + roi = roi.clone()
110 + if isinstance(key, torch.Tensor):
111 + key = key.clone()
112 + if isinstance(value, torch.Tensor):
113 + value = value.clone()
114 + if isinstance(hidden, torch.Tensor):
115 + hidden = hidden.clone()
116 +
117 + buffer_item = [input_tokens, roi, key, value, hidden]
118 +
119 + with self.buffer_lock:
120 + for data in buffer_item:
121 + self.buffer_size += self._get_element_size(data)
122 + self.buffer.append(buffer_item)
123 +
124 + def _is_end_signal(self, signal):
125 + return signal is None
126 +
127 + def drop_select_handler(self):
128 +
129 + try:
130 +
131 + while True:
132 + signal = self.signal_pipe.recv_tensor()
133 + if self._is_end_signal(signal):
134 + logger.info("Received end signal!")
135 + break
136 +
137 + input_tokens = self.data_pipe.recv_tensor()
138 +
139 + roi = self.data_pipe.recv_tensor()
140 + assert roi is not None, "Please provide the roi when sending "\
141 + "drop-select request"
142 + roi = (roi > 0.5)
143 + tokens_roi_recver = [input_tokens, roi]
144 +
145 + matched_length = 0
146 +
147 + # perform input tokens and roi matching
148 + # FIXME: this matching is O(n), ideally it should be O(1)
149 + # but this buffer size won't (and shouldn't) be too large so
150 + # the fix is not urgent.
151 + with self.buffer_lock:
152 +
153 + for _ in range(len(self.buffer)):
154 +
155 + temp_length = self._matches(self.buffer[0],
156 + tokens_roi_recver)
157 + if temp_length > 0:
158 + matched_length = temp_length
159 + break
160 + # rotate the element we just accessed to the end
161 + self.buffer.rotate(-1)
162 +
163 + if matched_length > 0:
164 + # need to clone the tensor
165 + # in case the tensor is freed before sending finishes
166 + matched_item = self.buffer.popleft()
167 + for tensor in matched_item:
168 + self._send_tensor_and_dec_size(tensor)
169 +
170 + else:
171 + # no match, just send None
172 + for _ in range(5):
173 + self.data_pipe.send_tensor(None)
174 +
175 + except RuntimeError as e:
176 + if 'Connection closed by peer' not in str(e):
177 + raise e
178 +
179 + logger.debug("Closing drop_select_handler")
180 +
181 + def drop_select(
182 + self, input_tokens: Optional[torch.Tensor],
183 + roi: Optional[torch.Tensor]) -> List[Optional[torch.Tensor]]:
184 +
185 + assert self.request_handling_thread is None, \
186 + "drop_select should be called by the KV cache consumer "\
187 + "(e.g. the decode vLLM instance)"
188 +
189 + if isinstance(input_tokens, torch.Tensor):
190 + input_tokens = input_tokens.clone()
191 + if isinstance(roi, torch.Tensor):
192 + roi = roi.clone().float()
193 +
194 + self.signal_pipe.send_tensor(self.normal_signal)
195 + self.data_pipe.send_tensor(input_tokens)
196 + self.data_pipe.send_tensor(roi)
197 +
198 + input_tokens = self.data_pipe.recv_tensor()
199 + roi = self.data_pipe.recv_tensor()
200 + if roi is not None:
201 + # convert from float tensor to bool tensor
202 + # as PyNccl does not support sending bool tensor
203 + roi = (roi > 0.5)
204 + key = self.data_pipe.recv_tensor()
205 + value = self.data_pipe.recv_tensor()
206 + hidden = self.data_pipe.recv_tensor()
207 +
208 + return [input_tokens, roi, key, value, hidden]
209 +
210 + def full_handler(self):
211 + time.sleep(0.001)
212 +
213 + def insert(self, input_tokens: torch.Tensor, roi: torch.Tensor,
214 + key: torch.Tensor, value: torch.Tensor,
215 + hidden: torch.Tensor) -> None:
216 +
217 + if self.buffer_size > self.buffer_size_threshold:
218 + # log outside the while loop to avoid this message being logged
219 + # repeatedly.
220 + logger.debug("KV transfer buffer is full. Handling...")
221 + while self.buffer_size > self.buffer_size_threshold:
222 + self.full_handler()
223 +
224 + self._add_to_buffer(input_tokens, roi, key, value, hidden)
225 +
226 + # when calling the insert, the current process is a sender
227 + # need to launch the request handler and start listening to request.
228 + if self.request_handling_thread is None:
229 + self.request_handling_thread = threading.Thread(
230 + target=self.drop_select_handler)
231 + self.request_handling_thread.start()
232 +
233 + def close(self):
234 +
235 + if hasattr(self, "request_handling_thread"
236 + ) and self.request_handling_thread is not None:
237 + self.request_handling_thread.join()
238 +
239 + else:
240 + # TODO: have a explicit close signal and have a explicit way to
241 + # check if it's requester
242 + self.signal_pipe.send_tensor(self.end_signal)
vllm/distributed/kv_transfer/kv_pipe/__init__.py
vllm/distributed/kv_transfer/kv_pipe/base.py new
+65
@@ -0,0 +1,65 @@
1 +"""
2 +This file defines an interface `KVPipeBase`
3 +that provides an abstraction for sending and receiving tensors, or None, via
4 +distributed communications.
5 +
6 +All classes instantiated from this interface are assumed to be a FIFO pipe.
7 +
8 +If your distributed communication platform already supports key-value lookup,
9 +you can bypass this interface and directly start from `kv_lookup_buffer`.
10 +"""
11 +
12 +from abc import ABC, abstractmethod
13 +from typing import Optional
14 +
15 +import torch
16 +
17 +
18 +class KVPipeBase(ABC):
19 + """
20 + This class provides an interface for sending and receiving tensors, or
21 + None, by distributed communications.
22 + """
23 +
24 + @abstractmethod
25 + def send_tensor(self, tensor: Optional[torch.Tensor]) -> None:
26 + """Send a tensor, or None, via the pipe.
27 +
28 + Need to support sending None -- important for error handling.
29 +
30 + TODO: add a `key` argument so that we can use traditional
31 + key-value database as the distributed communication mechanism behind
32 + the pipe.
33 +
34 + Args:
35 + tensor (Optional[torch.Tensor]): The tensor to be sent. Can be None.
36 +
37 + Raises:
38 + NotImplementedError: This method must be implemented in subclasses.
39 + """
40 + raise NotImplementedError
41 +
42 + @abstractmethod
43 + def recv_tensor(self) -> Optional[torch.Tensor]:
44 + """Receive a tensor (can be None) from the pipeline.
45 +
46 + Returns:
47 + Optional[torch.Tensor]: The tensor received from the pipeline. Can
48 + be None.
49 +
50 + Raises:
51 + NotImplementedError: This method must be implemented in subclasses.
52 + """
53 + raise NotImplementedError
54 +
55 + @abstractmethod
56 + def close(self) -> None:
57 + """Close the pipeline and release resources.
58 +
59 + This method is responsible for closing the communication pipeline
60 + and releasing any resources associated with it.
61 +
62 + Raises:
63 + NotImplementedError: This method must be implemented in subclasses.
64 + """
65 + raise NotImplementedError
vllm/distributed/kv_transfer/kv_pipe/pynccl_pipe.py new
+276
@@ -0,0 +1,276 @@
1 +"""
2 + This module implements a PyNccl pipe for sending and receiving
3 + Optional[torch.Tensor] between distributed ranks with advanced
4 + communication features.
5 +
6 + Key Features:
7 + - Supports sending and receiving tensors with metadata
8 + - Handles both CUDA and CPU device communications
9 + - Implements a non-blocking tensor transfer mechanism
10 + - Manages buffer size and provides backpressure control
11 + - Supports distributed process groups with configurable parameters
12 +"""
13 +
14 +import threading
15 +import time
16 +from concurrent.futures import ThreadPoolExecutor
17 +from typing import Callable, Dict, Optional, Tuple
18 +
19 +import torch
20 +
21 +from vllm.config import KVTransferConfig
22 +from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
23 +from vllm.distributed.kv_transfer.kv_pipe.base import KVPipeBase
24 +from vllm.distributed.utils import StatelessProcessGroup
25 +from vllm.logger import init_logger
26 +
27 +logger = init_logger(__name__)
28 +
29 +
30 +class BrokenPipeException(Exception):
31 +
32 + def __init__(self, message):
33 + self.message = message
34 + super().__init__(self.message)
35 +
36 +
37 +Metadata = Dict[str, Optional[torch.Tensor]]
38 +
39 +
40 +class PyNcclPipe(KVPipeBase):
41 +
42 + METADATA_LENGTH = 16
43 + MAX_TENSOR_DIMENSIONS = 14
44 + METADATA_DTYPE = torch.int64
45 +
46 + def __init__(self,
47 + local_rank: int,
48 + config: KVTransferConfig,
49 + device: Optional[str] = None,
50 + port_offset: int = 0):
51 + self.config = config
52 + self.local_rank = local_rank
53 + self.kv_rank = self.config.kv_rank
54 + self.kv_parallel_size = self.config.kv_parallel_size
55 + if device is None:
56 + self.device = self._select_device(self.config.kv_buffer_device)
57 + else:
58 + self.device = self._select_device(device)
59 +
60 + # build distributed connection and send/recv implementation
61 + self.group = StatelessProcessGroup.create(
62 + host=self.config.kv_ip,
63 + port=self.config.kv_port + port_offset,
64 + rank=self.kv_rank,
65 + world_size=self.kv_parallel_size,
66 + )
67 + # add a barrier to make sure the connection is initiated properly
68 + self.group.barrier()
69 + impl = self._get_device_send_recv_impl(self.group)
70 + self.device_send_func, self.device_recv_func = impl
71 + # set target rank
72 + self.target_rank_for_send = (self.kv_rank + 1) % self.kv_parallel_size
73 + self.target_rank_for_recv = (self.kv_rank - 1) % self.kv_parallel_size
74 +
75 + # transportation-related variables
76 + self.transport_thread: Optional[ThreadPoolExecutor] = None
77 + self.buffer_size = 0
78 + self.buffer_size_lock = threading.Lock()
79 + self.buffer_size_thresh = self.config.kv_buffer_size
80 +
81 + def _get_device_send_recv_impl(
82 + self, group: StatelessProcessGroup
83 + ) -> Tuple[Callable[[torch.Tensor, int], None], Callable[
84 + [torch.Tensor, int], None]]:
85 +
86 + send: Callable[[torch.Tensor, int], None]
87 + recv: Callable[[torch.Tensor, int], None]
88 + if self.device.type == "cuda":
89 + # use PyNCCL for send / recv
90 + comm = PyNcclCommunicator(group, device=self.local_rank)
91 + comm.disabled = False
92 + send, recv = comm.send, comm.recv # type: ignore
93 + else:
94 + # This send / recv implementation here is NOT intended to transfer
95 + # KV caches (and should NOT be repurposed to transfer KV caches).
96 + # Currently it is only used to transmit control-plane messages
97 + # for PyNcclBuffer.
98 + send = group.send_obj
99 +
100 + def my_recv(x, src):
101 + x[...] = group.recv_obj(src)
102 +
103 + recv = my_recv
104 +
105 + return send, recv
106 +
107 + def _select_device(self, device: str):
108 + logger.info("Selecting device: %s", device)
109 + if device == "cuda":
110 + return torch.device(f"cuda:{self.local_rank}")
111 + else:
112 + return torch.device("cpu")
113 +
114 + def _make_metadata(self, tensor: Optional[torch.Tensor]) -> Metadata:
115 + """
116 + Create the metadata as a dictionary based on the input tensor.
117 +
118 + Parameters:
119 + - tensor: The input tensor or None if no tensor is provided.
120 +
121 + Returns:
122 + - metadata: A dictionary with the following keys:
123 + - "dtype": The data type of the tensor or None.
124 + - "shape": The shape of the tensor or None.
125 + """
126 + if tensor is None:
127 + return {"dtype": None, "shape": None}
128 + else:
129 + return {"dtype": tensor.dtype, "shape": tensor.shape}
130 +
131 + def _prepare_recv_buffer(self, metadata: Metadata) -> torch.Tensor:
132 + """
133 + Create a buffer to receive the tensor based on the provided metadata.
134 +
135 + Parameters:
136 + - metadata: A dictionary with keys "dtype" and "shape", describing
137 + the tensor's data type and shape.
138 +
139 + Returns:
140 + - buffer: A tensor of the specified type and shape, allocated on
141 + self.device.
142 + """
143 + return torch.empty(metadata["shape"],
144 + dtype=metadata["dtype"],
145 + device=self.device)
146 +
147 + def _send_metadata(self, metadata: Metadata):
148 + """
149 + Send the metadata dictionary to the target rank.
150 +
151 + Parameters:
152 + - metadata: A dictionary with keys "dtype" and "shape".
153 + """
154 + self.group.send_obj(metadata, self.target_rank_for_send)
155 +
156 + def _recv_metadata(self) -> Metadata:
157 + """
158 + Receive the metadata dictionary from the target rank.
159 +
160 + Returns:
161 + - metadata: A dictionary with keys "dtype" and "shape" describing
162 + the tensor.
163 + """
164 + return self.group.recv_obj(self.target_rank_for_recv)
165 +
166 + def _send_impl(self, tensor: Optional[torch.Tensor]) -> None:
167 + """
168 + The actual implementation of sending the tensor and its metadata to the
169 + target rank.
170 +
171 + Parameters:
172 + - tensor: The input tensor to be sent, or None if no tensor is
173 + being sent.
174 + """
175 + metadata = self._make_metadata(tensor)
176 + self._send_metadata(metadata)
177 + if tensor is not None:
178 + self.device_send_func(tensor.to(self.device),
179 + self.target_rank_for_send)
180 +
181 + def _recv_impl(self) -> Optional[torch.Tensor]:
182 + """
183 + The actual implementation of receiving a tensor and its metadata from
184 + the target rank.
185 +
186 + Returns:
187 + - buffer: The received tensor, or None if no tensor is received.
188 + """
189 + metadata = self._recv_metadata()
190 + if metadata["dtype"] is None:
191 + return None
192 + buffer = self._prepare_recv_buffer(metadata)
193 + self.device_recv_func(buffer, self.target_rank_for_recv)
194 +
195 + return buffer
196 +
197 + def send_tensor_wrapper(self, tensor: Optional[torch.Tensor],
198 + tensor_size: int) -> None:
199 + """
200 + Wrapper for _send_impl to handle exceptions and update buffer size.
201 + """
202 + try:
203 + self._send_impl(tensor)
204 +
205 + with self.buffer_size_lock:
206 + self.buffer_size -= tensor_size
207 + except Exception as e:
208 + logger.error("[rank%d]: Exception when trying to send %s, msg: %s",
209 + torch.distributed.get_rank(), str(tensor), str(e))
210 + import traceback
211 + traceback.print_exc()
212 +
213 + def block_if_full(self):
214 + """
215 + Block the current thread if the buffer size is larger than the
216 + threshold.
217 + """
218 + while self.buffer_size > self.buffer_size_thresh:
219 + logger.debug("KV cache transfer pipe is full. Waiting...")
220 + time.sleep(0.05)
221 +
222 + def send_tensor(self, tensor: Optional[torch.Tensor]) -> None:
223 + """
224 + Sends a tensor and its metadata to the destination rank in a
225 + non-blocking way.
226 +
227 + Parameters:
228 + - tensor: The tensor to send, or None if no tensor is being sent.
229 + """
230 + if self.transport_thread is None:
231 + self.transport_thread = ThreadPoolExecutor(max_workers=1)
232 +
233 + if tensor is not None:
234 + tensor_size = tensor.element_size() * tensor.numel()
235 + else:
236 + tensor_size = 0
237 +
238 + self.block_if_full()
239 +
240 + with self.buffer_size_lock:
241 + self.buffer_size += tensor_size
242 +
243 + self.transport_thread.submit(self.send_tensor_wrapper, tensor,
244 + tensor_size)
245 +
246 + def recv_tensor(self) -> Optional[torch.Tensor]:
247 + """
248 + Receives a tensor and its metadata from the source rank. Blocking call.
249 +
250 + Returns:
251 + - tensor: The received tensor, or None if no tensor is received.
252 + """
253 + if self.transport_thread is None:
254 + self.transport_thread = ThreadPoolExecutor(max_workers=1)
255 +
256 + future = self.transport_thread.submit(self._recv_impl)
257 +
258 + try:
259 + tensor = future.result()
260 + except Exception as e:
261 + logger.error("Encountering exception in KV receiving thread")
262 + logger.error("%s", e)
263 + logger.error("My device: %s", self.device)
264 + import traceback
265 + traceback.print_exc()
266 + raise e
267 +
268 + return tensor
269 +
270 + def close(self):
271 + """
272 + Close the pipe and release associated resources.
273 + """
274 + if hasattr(self,
275 + "transport_thread") and self.transport_thread is not None:
276 + self.transport_thread.shutdown()
vllm/distributed/kv_transfer/kv_transfer_agent.py new
+75
@@ -0,0 +1,75 @@
1 +"""A centralized entrypoint to perform distributed KV cache transfer.
2 +
3 +This implementation is a shim wrapper on two APIs exposed by `kv_connector`:
4 +1. `send_kv_caches_and_hidden_states`
5 +2. `recv_kv_caches_and_hidden_states
6 +"""
7 +from typing import TYPE_CHECKING, List, Tuple, Union
8 +
9 +if TYPE_CHECKING:
10 + from vllm.worker.model_runner import ModelInputForGPUWithSamplingMetadata
11 + from vllm.config import VllmConfig
12 +
13 +import torch
14 +
15 +from vllm.distributed.kv_transfer.kv_connector.factory import (
16 + KVConnectorFactory)
17 +from vllm.logger import init_logger
18 +from vllm.sequence import IntermediateTensors
19 +
20 +logger = init_logger(__name__)
21 +
22 +
23 +class KVTransferAgent:
24 + """
25 + A class designated for distributed KV transfer
26 +
27 + Target use cases:
28 + 1. Disaggregated prefill
29 + 2. Remote KV cache storage
30 + """
31 +
32 + def __init__(
33 + self,
34 + rank: int,
35 + local_rank: int,
36 + config: "VllmConfig",
37 + ):
38 +
39 + self.config = config
40 +
41 + if config.kv_transfer_config is None:
42 + raise ValueError("KVTransferConfig is not set in the VllmConfig,"
43 + " cannot initialize KVConnector.")
44 +
45 + assert self.config.kv_transfer_config.is_kv_transfer_instance, "KV"\
46 + "TransferAgent should only be used when kv_connector is set."
47 +
48 + self.connector = KVConnectorFactory.create_connector(
49 + rank, local_rank, config)
50 +
51 + def send_kv_caches_and_hidden_states(
52 + self,
53 + model_executable: torch.nn.Module,
54 + model_input: "ModelInputForGPUWithSamplingMetadata",
55 + kv_caches: List[torch.Tensor],
56 + hidden_or_intermediate_states: Union[torch.Tensor,
57 + IntermediateTensors],
58 + ) -> None:
59 +
60 + self.connector.send_kv_caches_and_hidden_states(
61 + model_executable, model_input, kv_caches,
62 + hidden_or_intermediate_states)
63 +
64 + def close(self) -> None:
65 + self.connector.close()
66 +
67 + def recv_kv_caches_and_hidden_states(
68 + self, model_executable: torch.nn.Module,
69 + model_input: "ModelInputForGPUWithSamplingMetadata",
70 + kv_caches: List[torch.Tensor]
71 + ) -> Tuple[Union[torch.Tensor, IntermediateTensors], bool,
72 + "ModelInputForGPUWithSamplingMetadata"]:
73 +
74 + return self.connector.recv_kv_caches_and_hidden_states(
75 + model_executable, model_input, kv_caches)
vllm/distributed/parallel_state.py
+34 -1
@@ -27,18 +27,23 @@ from collections import namedtuple
27 from contextlib import contextmanager, nullcontext
28 from dataclasses import dataclass
29 from multiprocessing import shared_memory
30 -from typing import Any, Callable, Dict, List, Optional, Tuple, Union
30 +from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
31 + Union)
32 from unittest.mock import patch
33
34 import torch
35 import torch.distributed
36 from torch.distributed import Backend, ProcessGroup
37
38 +import vllm.distributed.kv_transfer.kv_transfer_agent as kv_transfer
39 import vllm.envs as envs
40 from vllm.logger import init_logger
41 from vllm.platforms import current_platform
42 from vllm.utils import direct_register_custom_op, supports_custom_op
43
44 +if TYPE_CHECKING:
45 + from vllm.config import VllmConfig
46 +
47
48 @dataclass
49 class GraphCaptureContext:
@@ -904,6 +909,14 @@ def get_pp_group() -> GroupCoordinator:
909 # kept for backward compatibility
910 get_pipeline_model_parallel_group = get_pp_group
911
912 +_KV_TRANSFER: Optional[kv_transfer.KVTransferAgent] = None
913 +
914 +
915 +def get_kv_transfer_group() -> kv_transfer.KVTransferAgent:
916 + assert _KV_TRANSFER is not None, (
917 + "disaggregated KV cache transfer parallel group is not initialized")
918 + return _KV_TRANSFER
919 +
920
921 @contextmanager
922 def graph_capture():
@@ -1052,6 +1065,26 @@ def initialize_model_parallel(
1065 group_name="pp")
1066
1067
1068 +def ensure_kv_transfer_initialized(vllm_config: "VllmConfig") -> None:
1069 + """
1070 + Initialize KV cache transfer parallel group.
1071 + """
1072 +
1073 + global _KV_TRANSFER
1074 +
1075 + if vllm_config.kv_transfer_config is None:
1076 + return
1077 +
1078 + if all([
1079 + vllm_config.kv_transfer_config.need_kv_parallel_group,
1080 + _KV_TRANSFER is None
1081 + ]):
1082 + _KV_TRANSFER = kv_transfer.KVTransferAgent(
1083 + rank=get_world_group().rank,
1084 + local_rank=get_world_group().local_rank,
1085 + config=vllm_config)
1086 +
1087 +
1088 def ensure_model_parallel_initialized(
1089 tensor_model_parallel_size: int,
1090 pipeline_model_parallel_size: int,
vllm/engine/arg_utils.py
+14 -4
@@ -9,10 +9,10 @@ import torch
9
10 import vllm.envs as envs
11 from vllm.config import (CacheConfig, CompilationConfig, ConfigFormat,
12 - DecodingConfig, DeviceConfig, HfOverrides, LoadConfig,
13 - LoadFormat, LoRAConfig, ModelConfig,
14 - ObservabilityConfig, ParallelConfig, PoolerConfig,
15 - PromptAdapterConfig, SchedulerConfig,
12 + DecodingConfig, DeviceConfig, HfOverrides,
13 + KVTransferConfig, LoadConfig, LoadFormat, LoRAConfig,
14 + ModelConfig, ObservabilityConfig, ParallelConfig,
15 + PoolerConfig, PromptAdapterConfig, SchedulerConfig,
16 SpeculativeConfig, TaskOption, TokenizerPoolConfig,
17 VllmConfig)
18 from vllm.executor.executor_base import ExecutorBase
@@ -108,6 +108,7 @@ class EngineArgs:
108 # notice.
109 distributed_executor_backend: Optional[Union[str,
110 Type[ExecutorBase]]] = None
111 + # number of P/D disaggregation (or other disaggregation) workers
112 pipeline_parallel_size: int = 1
113 tensor_parallel_size: int = 1
114 max_parallel_loading_workers: Optional[int] = None
@@ -194,6 +195,8 @@ class EngineArgs:
195 compilation_config: Optional[CompilationConfig] = None
196 worker_cls: str = "auto"
197
198 + kv_transfer_config: Optional[KVTransferConfig] = None
199 +
200 def __post_init__(self):
201 if not self.tokenizer:
202 self.tokenizer = self.model
@@ -908,6 +911,12 @@ class EngineArgs:
911 'compilers, using -O without space is also '
912 'supported. -O3 is equivalent to -O 3.')
913
914 + parser.add_argument('--kv-transfer-config',
915 + type=KVTransferConfig.from_cli,
916 + default=None,
917 + help='The configurations for distributed KV cache '
918 + 'transfer. Should be a JSON string.')
919 +
920 parser.add_argument(
921 '--worker-cls',
922 type=str,
@@ -1201,6 +1210,7 @@ class EngineArgs:
1210 observability_config=observability_config,
1211 prompt_adapter_config=prompt_adapter_config,
1212 compilation_config=self.compilation_config,
1213 + kv_transfer_config=self.kv_transfer_config,
1214 )
1215
1216 if envs.VLLM_USE_V1:
vllm/worker/model_runner.py
+94 -11
@@ -21,7 +21,7 @@ from vllm.attention.backends.utils import CommonAttentionState
21 from vllm.compilation.compile_context import set_compile_context
22 from vllm.config import CompilationLevel, VllmConfig
23 from vllm.core.scheduler import SchedulerOutputs
24 -from vllm.distributed import get_pp_group
24 +from vllm.distributed import get_kv_transfer_group, get_pp_group
25 from vllm.distributed.parallel_state import graph_capture
26 from vllm.forward_context import set_forward_context
27 from vllm.inputs import INPUT_REGISTRY, InputRegistry
@@ -1666,6 +1666,24 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1666 else:
1667 model_executable = self.model
1668
1669 + # Receive KV cache in distributed KV cache transfer setting
1670 + # In disagg prefill setting, it will also recv hidden states and bypass
1671 + # model forwarding
1672 + # In KV cache database setting, it will change the model input so that
1673 + # we can skip prefilling on tokens that successfully received KV caches
1674 + # NOTE: The receive operation is blocking
1675 + bypass_model_exec = False
1676 + if self.need_recv_kv(model_input, kv_caches):
1677 + hidden_or_intermediate_states, bypass_model_exec, model_input = \
1678 + get_kv_transfer_group().recv_kv_caches_and_hidden_states(
1679 + # model is used to know which layer the current worker
1680 + # is working on, so that we can receive KV for only those
1681 + # layers.
1682 + model_executable,
1683 + model_input,
1684 + kv_caches=kv_caches
1685 + )
1686 +
1687 multi_modal_kwargs = model_input.multi_modal_kwargs or {}
1688 seqlen_agnostic_kwargs = {
1689 "finished_requests_ids": model_input.finished_requests_ids,
@@ -1677,21 +1695,36 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1695 model_forward_end = torch.cuda.Event(enable_timing=True)
1696 model_forward_start.record()
1697
1680 - with set_forward_context(model_input.attn_metadata, self.vllm_config):
1681 - hidden_or_intermediate_states = model_executable(
1682 - input_ids=model_input.input_tokens,
1683 - positions=model_input.input_positions,
1684 - kv_caches=kv_caches,
1685 - attn_metadata=model_input.attn_metadata,
1686 - intermediate_tensors=intermediate_tensors,
1687 - **MultiModalKwargs.as_kwargs(multi_modal_kwargs,
1688 - device=self.device),
1689 - **seqlen_agnostic_kwargs)
1698 + if not bypass_model_exec:
1699 + with set_forward_context(model_input.attn_metadata,
1700 + self.vllm_config):
1701 + hidden_or_intermediate_states = model_executable(
1702 + input_ids=model_input.input_tokens,
1703 + positions=model_input.input_positions,
1704 + kv_caches=kv_caches,
1705 + attn_metadata=model_input.attn_metadata,
1706 + intermediate_tensors=intermediate_tensors,
1707 + **MultiModalKwargs.as_kwargs(multi_modal_kwargs,
1708 + device=self.device),
1709 + **seqlen_agnostic_kwargs)
1710
1711 if (self.observability_config is not None
1712 and self.observability_config.collect_model_forward_time):
1713 model_forward_end.record()
1714
1715 + # Sending KV cache in distributed KV cache transfer setting
1716 + # NOTE: the send operation is non-blocking
1717 + if self.need_send_kv(model_input, kv_caches):
1718 + get_kv_transfer_group().send_kv_caches_and_hidden_states(
1719 + # model_executable is used to know which layer the current
1720 + # worker is working on, so that we can send KV for only those
1721 + # layers.
1722 + model_executable,
1723 + model_input,
1724 + kv_caches,
1725 + hidden_or_intermediate_states,
1726 + )
1727 +
1728 # Compute the logits in the last pipeline stage.
1729 if not get_pp_group().is_last_rank:
1730 if (self.is_driver_worker
@@ -1759,6 +1792,56 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1792
1793 return [output]
1794
1795 + def need_recv_kv(self, model_input, kv_caches) -> bool:
1796 + """Check if we need to receive kv-cache from the other worker.
1797 + We need to receive KV when
1798 + 1. current vLLM instance is KV cache consumer/decode vLLM instance
1799 + 2. this batch is not a profiling run
1800 + 3. this batch is a prefill run
1801 +
1802 + Args:
1803 + model_input: input to the model executable
1804 + kv_caches: vLLM's paged memory
1805 + """
1806 +
1807 + prefill_meta = model_input.attn_metadata.prefill_metadata
1808 +
1809 + # check if the current run is profiling
1810 + is_profile_run = (kv_caches[0].numel() == 0)
1811 + # check if the current run is prefill
1812 + is_prefill_run = prefill_meta is not None
1813 +
1814 + if self.vllm_config.kv_transfer_config is None:
1815 + return False
1816 +
1817 + return self.vllm_config.kv_transfer_config.is_kv_consumer and (
1818 + not is_profile_run) and is_prefill_run
1819 +
1820 + def need_send_kv(self, model_input, kv_caches) -> bool:
1821 + """Check if we need to send kv-cache to the other worker.
1822 + We need to send KV when
1823 + 1. current vLLM instance is KV cache producer/prefill vLLM instance
1824 + 2. this batch is not a profiling run
1825 + 3. this batch is a prefill run
1826 +
1827 + Args:
1828 + model_input: input to the model executable
1829 + kv_caches: vLLM's paged memory
1830 + """
1831 +
1832 + prefill_meta = model_input.attn_metadata.prefill_metadata
1833 +
1834 + # check if the current run is profiling
1835 + is_profile_run = (kv_caches[0].numel() == 0)
1836 + # check if the current run is prefill
1837 + is_prefill_run = prefill_meta is not None
1838 +
1839 + if self.vllm_config.kv_transfer_config is None:
1840 + return False
1841 +
1842 + return self.vllm_config.kv_transfer_config.is_kv_producer and (
1843 + not is_profile_run) and is_prefill_run
1844 +
1845
1846 # NOTE: this is nn.Module so the profiler can properly capture/group
1847 # kernels calls made within the graph
vllm/worker/worker.py
+8 -5
@@ -8,8 +8,9 @@ import torch
8 import torch.distributed
9
10 import vllm.envs as envs
11 -from vllm.config import ParallelConfig, VllmConfig
12 -from vllm.distributed import (ensure_model_parallel_initialized,
11 +from vllm.config import VllmConfig
12 +from vllm.distributed import (ensure_kv_transfer_initialized,
13 + ensure_model_parallel_initialized,
14 init_distributed_environment,
15 set_custom_all_reduce)
16 from vllm.logger import init_logger
@@ -144,7 +145,7 @@ class Worker(LocalOrDistributedWorkerBase):
145 raise RuntimeError(
146 f"Not support device type: {self.device_config.device}")
147 # Initialize the distributed environment.
147 - init_worker_distributed_environment(self.parallel_config, self.rank,
148 + init_worker_distributed_environment(self.vllm_config, self.rank,
149 self.distributed_init_method,
150 self.local_rank)
151 # Set random seed.
@@ -457,20 +458,22 @@ class Worker(LocalOrDistributedWorkerBase):
458
459
460 def init_worker_distributed_environment(
460 - parallel_config: ParallelConfig,
461 + vllm_config: VllmConfig,
462 rank: int,
463 distributed_init_method: Optional[str] = None,
464 local_rank: int = -1,
465 ) -> None:
466 """Initialize the distributed environment."""
467 + parallel_config = vllm_config.parallel_config
468 set_custom_all_reduce(not parallel_config.disable_custom_all_reduce)
469
470 init_distributed_environment(parallel_config.world_size, rank,
471 distributed_init_method, local_rank)
470 -
472 ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
473 parallel_config.pipeline_parallel_size)
474
475 + ensure_kv_transfer_initialized(vllm_config)
476 +
477
478 def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
479 # Check if the GPU supports the dtype.
vllm/worker/worker_base.py
+1
@@ -43,6 +43,7 @@ class WorkerBase(ABC):
43 self.speculative_config = vllm_config.speculative_config
44 self.prompt_adapter_config = vllm_config.prompt_adapter_config
45 self.observability_config = vllm_config.observability_config
46 + self.kv_transfer_config = vllm_config.kv_transfer_config
47
48 @abstractmethod
49 def init_device(self) -> None: