A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Core] Pipeline Parallel Support (#4412)
Signed-off-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai>
Murali Andoorveedu committed
Jul 2, 2024 at 10:58 UTC
c5832d2ae9431a1672d547c232ec46b1a9051ff0
82 files changed
+1100
-404
.buildkite/test-pipeline.yaml
+10
@@ -74,6 +74,16 @@ steps:
74
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
75
- pytest -v -s spec_decode/e2e/test_integration_dist_tp4.py
76
77
+- label: Pipeline Parallelism Test
78
+ working_dir: "/vllm-workspace/tests"
79
+ num_gpus: 4
80
+ commands:
81
+ - TP_SIZE=2 PP_SIZE=2 EAGER_MODE=1 CHUNKED_PREFILL=1 pytest -v -s distributed/test_pipeline_parallel.py
82
+ - TP_SIZE=2 PP_SIZE=2 EAGER_MODE=1 CHUNKED_PREFILL=0 pytest -v -s distributed/test_pipeline_parallel.py
83
+ - PP_SIZE=4 EAGER_MODE=1 CHUNKED_PREFILL=1 pytest -v -s distributed/test_pipeline_parallel.py
84
+ - PP_SIZE=4 EAGER_MODE=1 CHUNKED_PREFILL=0 pytest -v -s distributed/test_pipeline_parallel.py
85
+
86
+
87
- label: Engine Test
88
mirror_hardwares: [amd]
89
command: pytest -v -s engine tokenization test_sequence.py test_config.py test_logger.py
tests/async_engine/test_async_llm_engine.py
+13
-1
@@ -5,6 +5,7 @@ import pytest
5
import torch
6
7
from vllm import SamplingParams
8
+from vllm.config import ParallelConfig
9
from vllm.engine.async_llm_engine import AsyncEngineArgs, AsyncLLMEngine
10
11
from ..utils import wait_for_gpu_memory_to_clear
@@ -23,8 +24,11 @@ class MockEngine:
24
self.add_request_calls = 0
25
self.abort_request_calls = 0
26
self.request_id = None
27
+ # Ugly, remove dependency when possible
28
+ self.parallel_config = ParallelConfig(1, 1, False)
29
27
- async def step_async(self):
30
+ async def step_async(self, virtual_engine):
31
+ # PP size is 1, ignore virtual engine
32
self.step_calls += 1
33
return [RequestOutput(
34
request_id=self.request_id)] if self.request_id else []
@@ -32,6 +36,9 @@ class MockEngine:
36
async def process_model_inputs_async(self, *args, **kwargs):
37
pass
38
39
+ async def stop_remote_worker_execution_loop_async(self):
40
+ pass
41
+
42
def generate(self, request_id):
43
self.request_id = request_id
44
@@ -41,6 +48,7 @@ class MockEngine:
48
def add_request(self, **kwargs):
49
del kwargs # Unused
50
self.add_request_calls += 1
51
+ print(f'Request calls: {self.add_request_calls}')
52
53
async def add_request_async(self, **kwargs):
54
self.add_request_calls += 1
@@ -53,6 +61,9 @@ class MockEngine:
61
def has_unfinished_requests(self):
62
return self.request_id is not None
63
64
+ def has_unfinished_requests_for_virtual_engine(self, virtual_engine):
65
+ return self.request_id is not None
66
+
67
68
class MockAsyncLLMEngine(AsyncLLMEngine):
69
@@ -76,6 +87,7 @@ async def test_new_requests_event():
87
engine.engine.generate("2")
88
await asyncio.sleep(0)
89
await asyncio.sleep(0)
90
+ await asyncio.sleep(0)
91
assert engine.engine.add_request_calls == 2
92
assert engine.engine.step_calls >= 2
93
await asyncio.sleep(0.001)
tests/async_engine/test_openapi_server_ray.py
+2
-2
@@ -4,7 +4,7 @@ import pytest
4
# and debugging.
5
import ray
6
7
-from ..utils import RemoteOpenAIServer
7
+from ..utils import VLLM_PATH, RemoteOpenAIServer
8
9
# any model with a chat template should work here
10
MODEL_NAME = "facebook/opt-125m"
@@ -12,7 +12,7 @@ MODEL_NAME = "facebook/opt-125m"
12
13
@pytest.fixture(scope="module")
14
def ray_ctx():
15
- ray.init()
15
+ ray.init(runtime_env={"working_dir": VLLM_PATH})
16
yield
17
ray.shutdown()
18
tests/basic_correctness/test_preemption.py
+12
-12
@@ -56,8 +56,8 @@ def test_chunked_prefill_recompute(
56
max_num_seqs=max_num_seqs,
57
) as vllm_model:
58
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
59
- assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
60
- ARTIFICIAL_PREEMPTION_MAX_CNT)
59
+ assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
60
+ < ARTIFICIAL_PREEMPTION_MAX_CNT)
61
62
for i in range(len(example_prompts)):
63
hf_output_ids, hf_output_str = hf_outputs[i]
@@ -91,10 +91,10 @@ def test_preemption(
91
disable_log_stats=False,
92
) as vllm_model:
93
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
94
- assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
95
- ARTIFICIAL_PREEMPTION_MAX_CNT)
94
+ assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
95
+ < ARTIFICIAL_PREEMPTION_MAX_CNT)
96
total_preemption = (
97
- vllm_model.model.llm_engine.scheduler.num_cumulative_preemption)
97
+ vllm_model.model.llm_engine.scheduler[0].num_cumulative_preemption)
98
99
check_outputs_equal(
100
outputs_0_lst=hf_outputs,
@@ -147,10 +147,10 @@ def test_swap(
147
) as vllm_model:
148
vllm_outputs = vllm_model.generate_beam_search(example_prompts,
149
beam_width, max_tokens)
150
- assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
151
- ARTIFICIAL_PREEMPTION_MAX_CNT)
150
+ assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
151
+ < ARTIFICIAL_PREEMPTION_MAX_CNT)
152
total_preemption = (
153
- vllm_model.model.llm_engine.scheduler.num_cumulative_preemption)
153
+ vllm_model.model.llm_engine.scheduler[0].num_cumulative_preemption)
154
155
for i in range(len(example_prompts)):
156
hf_output_ids, _ = hf_outputs[i]
@@ -214,8 +214,8 @@ def test_swap_infeasible(
214
example_prompts,
215
sampling_params=sampling_params,
216
)
217
- assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
218
- ARTIFICIAL_PREEMPTION_MAX_CNT)
217
+ assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
218
+ < ARTIFICIAL_PREEMPTION_MAX_CNT)
219
220
# Verify the request is ignored and not hang.
221
assert req_outputs[0].outputs[0].finish_reason == "length"
@@ -252,8 +252,8 @@ def test_preemption_infeasible(
252
sampling_params=sampling_params,
253
)
254
255
- assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
256
- ARTIFICIAL_PREEMPTION_MAX_CNT)
255
+ assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
256
+ < ARTIFICIAL_PREEMPTION_MAX_CNT)
257
258
# Verify the request is ignored and not hang.
259
for req_output in req_outputs:
tests/distributed/test_comm_ops.py
+17
-3
@@ -32,7 +32,7 @@ def all_reduce_test_worker(tp_size: int, pp_size: int, rank: int,
32
(r + 1) for r in range(tp_size)
33
]
34
expected = torch.sum(torch.stack(all_tensors, dim=0), dim=0)
35
- t = all_tensors[rank]
35
+ t = all_tensors[rank % tp_size]
36
t = tensor_model_parallel_all_reduce(t)
37
assert torch.allclose(t, expected)
38
@@ -60,7 +60,7 @@ def all_gather_test_worker(tp_size: int, pp_size: int, rank: int,
60
for r in range(tp_size)
61
]
62
expected = torch.cat(all_tensors, dim=all_gather_dimension)
63
- t = all_tensors[rank]
63
+ t = all_tensors[rank % tp_size]
64
t = tensor_model_parallel_all_gather(t, all_gather_dimension)
65
assert torch.allclose(t, expected)
66
@@ -91,7 +91,7 @@ def broadcast_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
91
"f": torch.tensor([], dtype=torch.float32, device="cuda"),
92
}
93
94
- if rank == 0:
94
+ if (rank % tp_size) == 0:
95
broadcast_tensor_dict(test_dict, src=0)
96
else:
97
recv_dict = broadcast_tensor_dict(src=0)
@@ -184,3 +184,17 @@ def test_multi_process_tensor_parallel(tp_size, test_target):
184
"test_target", [send_recv_test_worker, send_recv_tensor_dict_test_worker])
185
def test_multi_process_pipeline_parallel(pp_size, test_target):
186
multi_process_parallel(1, pp_size, test_target)
187
+
188
+
189
+@pytest.mark.skipif(torch.cuda.device_count() < 4,
190
+ reason="Need at least 4 GPUs to run the test.")
191
+@pytest.mark.parametrize("tp_size", [2])
192
+@pytest.mark.parametrize("pp_size", [2])
193
+@pytest.mark.parametrize("test_target", [
194
+ send_recv_test_worker, send_recv_tensor_dict_test_worker,
195
+ all_reduce_test_worker, all_gather_test_worker,
196
+ broadcast_tensor_dict_test_worker
197
+])
198
+def test_multi_process_tensor_parallel_pipeline_parallel(
199
+ tp_size, pp_size, test_target):
200
+ multi_process_parallel(tp_size, pp_size, test_target)
tests/distributed/test_pipeline_parallel.py
new
+149
@@ -0,0 +1,149 @@
1
+import os
2
+
3
+import openai # use the official client for correctness check
4
+import pytest
5
+# using Ray for overall ease of process management, parallel requests,
6
+# and debugging.
7
+import ray
8
+
9
+from ..utils import VLLM_PATH, RemoteOpenAIServer
10
+
11
+# downloading lora to test lora requests
12
+
13
+# any model with a chat template should work here
14
+MODEL_NAME = "meta-llama/Meta-Llama-3-8B"
15
+EAGER_MODE = bool(int(os.getenv("EAGER_MODE", 0)))
16
+CHUNKED_PREFILL = bool(int(os.getenv("CHUNKED_PREFILL", 0)))
17
+TP_SIZE = int(os.getenv("TP_SIZE", 1))
18
+PP_SIZE = int(os.getenv("PP_SIZE", 1))
19
+
20
+pytestmark = pytest.mark.asyncio
21
+
22
+
23
+@pytest.fixture(scope="module")
24
+def ray_ctx():
25
+ ray.init(runtime_env={"working_dir": VLLM_PATH})
26
+ yield
27
+ ray.shutdown()
28
+
29
+
30
+@pytest.fixture(scope="module")
31
+def server(ray_ctx):
32
+ args = [
33
+ "--model",
34
+ MODEL_NAME,
35
+ # use half precision for speed and memory savings in CI environment
36
+ "--dtype",
37
+ "bfloat16",
38
+ "--pipeline-parallel-size",
39
+ str(PP_SIZE),
40
+ "--tensor-parallel-size",
41
+ str(TP_SIZE),
42
+ "--distributed-executor-backend",
43
+ "ray",
44
+ ]
45
+ if CHUNKED_PREFILL:
46
+ args += [
47
+ "--enable-chunked-prefill",
48
+ ]
49
+ if EAGER_MODE:
50
+ args += [
51
+ "--enforce-eager",
52
+ ]
53
+ return RemoteOpenAIServer(args, num_gpus=PP_SIZE * TP_SIZE)
54
+
55
+
56
+@pytest.fixture(scope="module")
57
+def client(server):
58
+ return server.get_async_client()
59
+
60
+
61
+async def test_check_models(server, client: openai.AsyncOpenAI):
62
+ models = await client.models.list()
63
+ models = models.data
64
+ served_model = models[0]
65
+ assert served_model.id == MODEL_NAME
66
+ assert all(model.root == MODEL_NAME for model in models)
67
+
68
+
69
+@pytest.mark.parametrize(
70
+ "model_name",
71
+ [MODEL_NAME],
72
+)
73
+async def test_single_completion(server, client: openai.AsyncOpenAI,
74
+ model_name: str):
75
+ completion = await client.completions.create(model=model_name,
76
+ prompt="Hello, my name is",
77
+ max_tokens=5,
78
+ temperature=0.0)
79
+
80
+ assert completion.id is not None
81
+ assert completion.choices is not None and len(completion.choices) == 1
82
+ assert completion.choices[0].text is not None and len(
83
+ completion.choices[0].text) >= 5
84
+ assert completion.choices[0].finish_reason == "length"
85
+ assert completion.usage == openai.types.CompletionUsage(
86
+ completion_tokens=5, prompt_tokens=6, total_tokens=11)
87
+
88
+ # test using token IDs
89
+ completion = await client.completions.create(
90
+ model=MODEL_NAME,
91
+ prompt=[0, 0, 0, 0, 0],
92
+ max_tokens=5,
93
+ temperature=0.0,
94
+ )
95
+ assert completion.choices[0].text is not None and len(
96
+ completion.choices[0].text) >= 5
97
+
98
+
99
+@pytest.mark.parametrize(
100
+ # just test 1 lora hereafter
101
+ "model_name",
102
+ [MODEL_NAME],
103
+)
104
+async def test_batch_completions(server, client: openai.AsyncOpenAI,
105
+ model_name: str):
106
+ # test simple list
107
+ batch = await client.completions.create(
108
+ model=model_name,
109
+ prompt=["Hello, my name is", "Hello, my name is"],
110
+ max_tokens=5,
111
+ temperature=0.0,
112
+ )
113
+ assert len(batch.choices) == 2
114
+ assert batch.choices[0].text == batch.choices[1].text
115
+
116
+ # test n = 2
117
+ batch = await client.completions.create(
118
+ model=model_name,
119
+ prompt=["Hello, my name is", "Hello, my name is"],
120
+ n=2,
121
+ max_tokens=5,
122
+ temperature=0.0,
123
+ extra_body=dict(
124
+ # NOTE: this has to be true for n > 1 in vLLM, but not necessary
125
+ # for official client.
126
+ use_beam_search=True),
127
+ )
128
+ assert len(batch.choices) == 4
129
+ assert batch.choices[0].text != batch.choices[
130
+ 1].text, "beam search should be different"
131
+ assert batch.choices[0].text == batch.choices[
132
+ 2].text, "two copies of the same prompt should be the same"
133
+ assert batch.choices[1].text == batch.choices[
134
+ 3].text, "two copies of the same prompt should be the same"
135
+
136
+ # test streaming
137
+ batch = await client.completions.create(
138
+ model=model_name,
139
+ prompt=["Hello, my name is", "Hello, my name is"],
140
+ max_tokens=5,
141
+ temperature=0.0,
142
+ stream=True,
143
+ )
144
+ texts = [""] * 2
145
+ async for chunk in batch:
146
+ assert len(chunk.choices) == 1
147
+ choice = chunk.choices[0]
148
+ texts[choice.index] += choice.text
149
+ assert texts[0] == texts[1]
tests/engine/output_processor/test_multi_step.py
+4
-4
@@ -32,7 +32,7 @@ def test_appends_token_ids(num_new_tokens: int, seq_output_len: int):
32
33
output_processor = MultiStepOutputProcessor(
34
detokenizer=detokenizer,
35
- scheduler=scheduler,
35
+ scheduler=[scheduler],
36
seq_counter=seq_counter,
37
get_tokenizer_for_seq=lambda _: mock_tokenizer(),
38
stop_checker=stop_checker,
@@ -86,7 +86,7 @@ def test_respects_max_tokens(num_new_tokens: int, seq_prompt_len: int,
86
87
output_processor = MultiStepOutputProcessor(
88
detokenizer=detokenizer,
89
- scheduler=scheduler,
89
+ scheduler=[scheduler],
90
seq_counter=seq_counter,
91
get_tokenizer_for_seq=lambda _: mock_tokenizer(),
92
stop_checker=stop_checker,
@@ -148,7 +148,7 @@ def test_respects_eos_token_id(num_new_tokens: int, seq_prompt_len: int,
148
149
output_processor = MultiStepOutputProcessor(
150
detokenizer=detokenizer,
151
- scheduler=scheduler,
151
+ scheduler=[scheduler],
152
seq_counter=seq_counter,
153
get_tokenizer_for_seq=lambda _: mock_tokenizer(eos_token_id),
154
stop_checker=stop_checker,
@@ -215,7 +215,7 @@ def test_ignores_eos_token_id(num_new_tokens: int, seq_prompt_len: int,
215
216
output_processor = MultiStepOutputProcessor(
217
detokenizer=detokenizer,
218
- scheduler=scheduler,
218
+ scheduler=[scheduler],
219
seq_counter=seq_counter,
220
get_tokenizer_for_seq=lambda _: mock_tokenizer(eos_token_id),
221
stop_checker=stop_checker,
tests/entrypoints/openai/test_chat.py
+2
-2
@@ -14,7 +14,7 @@ import torch
14
from huggingface_hub import snapshot_download
15
from openai import BadRequestError
16
17
-from ...utils import RemoteOpenAIServer
17
+from ...utils import VLLM_PATH, RemoteOpenAIServer
18
19
# any model with a chat template should work here
20
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
@@ -77,7 +77,7 @@ def zephyr_lora_files():
77
78
@pytest.fixture(scope="module")
79
def ray_ctx():
80
- ray.init()
80
+ ray.init(runtime_env={"working_dir": VLLM_PATH})
81
yield
82
ray.shutdown()
83
tests/entrypoints/openai/test_completion.py
+2
-2
@@ -16,7 +16,7 @@ from openai import BadRequestError
16
17
from vllm.transformers_utils.tokenizer import get_tokenizer
18
19
-from ...utils import RemoteOpenAIServer
19
+from ...utils import VLLM_PATH, RemoteOpenAIServer
20
21
# any model with a chat template should work here
22
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
@@ -79,7 +79,7 @@ def zephyr_lora_files():
79
80
@pytest.fixture(scope="module")
81
def ray_ctx():
82
- ray.init()
82
+ ray.init(runtime_env={"working_dir": VLLM_PATH})
83
yield
84
ray.shutdown()
85
tests/entrypoints/openai/test_embedding.py
+2
-2
@@ -5,14 +5,14 @@ import openai
5
import pytest
6
import ray
7
8
-from ...utils import RemoteOpenAIServer
8
+from ...utils import VLLM_PATH, RemoteOpenAIServer
9
10
EMBEDDING_MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
11
12
13
@pytest.fixture(scope="module")
14
def ray_ctx():
15
- ray.init()
15
+ ray.init(runtime_env={"working_dir": VLLM_PATH})
16
yield
17
ray.shutdown()
18
tests/entrypoints/openai/test_models.py
+2
-2
@@ -6,7 +6,7 @@ import ray
6
# downloading lora to test lora requests
7
from huggingface_hub import snapshot_download
8
9
-from ...utils import RemoteOpenAIServer
9
+from ...utils import VLLM_PATH, RemoteOpenAIServer
10
11
# any model with a chat template should work here
12
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
@@ -22,7 +22,7 @@ def zephyr_lora_files():
22
23
@pytest.fixture(scope="module")
24
def ray_ctx():
25
- ray.init()
25
+ ray.init(runtime_env={"working_dir": VLLM_PATH})
26
yield
27
ray.shutdown()
28
tests/entrypoints/openai/test_vision.py
+2
-2
@@ -24,13 +24,13 @@ TEST_IMAGE_URLS = [
24
25
@pytest.fixture(scope="module")
26
def ray_ctx():
27
- ray.init()
27
+ ray.init(runtime_env={"working_dir": VLLM_PATH})
28
yield
29
ray.shutdown()
30
31
32
@pytest.fixture(scope="module")
33
-def server():
33
+def server(ray_ctx):
34
return RemoteOpenAIServer([
35
"--model",
36
MODEL_NAME,
tests/spec_decode/utils.py
+3
-3
@@ -54,9 +54,9 @@ def patch_execute_model_with_seeds(worker: Worker, rand_seeds: List[int]):
54
return new_execute_model
55
56
57
-def zero_kv_cache(cache_engine: CacheEngine):
58
- assert cache_engine.gpu_cache
59
- for key_blocks, value_blocks in cache_engine.gpu_cache:
57
+def zero_kv_cache(cache_engine: List[CacheEngine]):
58
+ assert cache_engine[0].gpu_cache
59
+ for key_blocks, value_blocks in cache_engine[0].gpu_cache:
60
key_blocks.zero_()
61
value_blocks.zero_()
62
tests/tensorizer_loader/test_tensorizer.py
+3
-1
@@ -22,7 +22,7 @@ from vllm.model_executor.model_loader.tensorizer import (TensorizerConfig,
22
tensorize_vllm_model)
23
24
from ..conftest import VllmRunner, cleanup
25
-from ..utils import RemoteOpenAIServer
25
+from ..utils import VLLM_PATH, RemoteOpenAIServer
26
27
# yapf conflicts with isort for this docstring
28
@@ -220,6 +220,8 @@ def test_openai_apiserver_with_tensorizer(vllm_runner, tmp_path):
220
json.dumps(model_loader_extra_config),
221
]
222
223
+ ray.init(runtime_env={"working_dir": VLLM_PATH})
224
+
225
server = RemoteOpenAIServer(openai_args)
226
print("Server ready.")
227
tests/utils.py
+10
-6
@@ -49,7 +49,6 @@ class RemoteOpenAIServer:
49
DUMMY_API_KEY = "token-abc123" # vLLM's OpenAI server does not need API key
50
MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
51
52
- @ray.remote(num_gpus=1)
52
class _RemoteRunner:
53
54
def __init__(self, cli_args: List[str], *, wait_url: str,
@@ -92,7 +91,11 @@ class RemoteOpenAIServer:
91
if hasattr(self, "proc"):
92
self.proc.terminate()
93
95
- def __init__(self, cli_args: List[str], *, auto_port: bool = True) -> None:
94
+ def __init__(self,
95
+ cli_args: List[str],
96
+ *,
97
+ auto_port: bool = True,
98
+ num_gpus: int = 1) -> None:
99
if auto_port:
100
if "-p" in cli_args or "--port" in cli_args:
101
raise ValueError("You have manually specified the port"
@@ -105,10 +108,11 @@ class RemoteOpenAIServer:
108
self.host = str(args.host or 'localhost')
109
self.port = int(args.port)
110
108
- self._runner = self._RemoteRunner.remote( # type: ignore
109
- cli_args,
110
- wait_url=self.url_for("health"),
111
- wait_timeout=self.MAX_SERVER_START_WAIT_S)
111
+ self._runner = ray.remote(num_gpus=num_gpus)(
112
+ self._RemoteRunner).remote(
113
+ cli_args,
114
+ wait_url=self.url_for("health"),
115
+ wait_timeout=self.MAX_SERVER_START_WAIT_S)
116
117
self._wait_until_ready()
118
tests/worker/test_swap.py
+2
-2
@@ -39,8 +39,8 @@ def test_swap() -> None:
39
num_cpu_blocks=engine_config.cache_config.num_cpu_blocks)
40
41
# Randomly initialize the cache.
42
- gpu_cache = worker.cache_engine.gpu_cache
43
- cpu_cache = worker.cache_engine.cpu_cache
42
+ gpu_cache = worker.cache_engine[0].gpu_cache
43
+ cpu_cache = worker.cache_engine[0].cpu_cache
44
num_layers = len(gpu_cache)
45
for i in range(num_layers):
46
gpu_key_cache, gpu_value_cache = gpu_cache[i]
vllm/config.py
+22
-3
@@ -27,6 +27,17 @@ logger = init_logger(__name__)
27
_GB = 1 << 30
28
_EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
29
30
+_PP_SUPPORTED_MODELS = [
31
+ "AquilaModel",
32
+ "AquilaForCausalLM",
33
+ "InternLMForCausalLM",
34
+ "LlamaForCausalLM",
35
+ "LLaMAForCausalLM",
36
+ "MistralForCausalLM",
37
+ "Phi3ForCausalLM",
38
+ "GPT2LMHeadModel",
39
+]
40
+
41
42
class ModelConfig:
43
"""Configuration for the model.
@@ -258,6 +269,13 @@ class ModelConfig:
269
total_num_hidden_layers = getattr(self.hf_text_config,
270
"num_hidden_layers", 0)
271
pipeline_parallel_size = parallel_config.pipeline_parallel_size
272
+ architectures = getattr(self.hf_config, "architectures", [])
273
+ if not all(arch in _PP_SUPPORTED_MODELS
274
+ for arch in architectures) and pipeline_parallel_size > 1:
275
+ raise NotImplementedError(
276
+ "Pipeline parallelism is only supported for the following "
277
+ f" architectures: {_PP_SUPPORTED_MODELS}.")
278
+
279
if total_num_hidden_layers % pipeline_parallel_size != 0:
280
raise ValueError(
281
f"Total number of hidden layers ({total_num_hidden_layers}) "
@@ -665,9 +683,10 @@ class ParallelConfig:
683
self._verify_args()
684
685
def _verify_args(self) -> None:
668
- if self.pipeline_parallel_size > 1:
669
- raise NotImplementedError(
670
- "Pipeline parallelism is not supported yet.")
686
+ if (self.pipeline_parallel_size > 1
687
+ and self.distributed_executor_backend == "mp"):
688
+ raise NotImplementedError("Pipeline parallelism is not supported "
689
+ "yet with multiprocessing.")
690
if self.distributed_executor_backend not in ("ray", "mp", None):
691
raise ValueError(
692
"Unrecognized distributed executor backend. Supported values "
vllm/core/block_manager_v1.py
+3
@@ -471,6 +471,9 @@ class BlockSpaceManagerV1(BlockSpaceManager):
471
def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
472
# NOTE: fork does not allocate a new physical block.
473
# Thus, it is always safe from OOM.
474
+ if parent_seq.seq_id not in self.block_tables:
475
+ # Parent sequence has either been freed or never existed.
476
+ return
477
src_block_table = self.block_tables[parent_seq.seq_id]
478
self.block_tables[child_seq.seq_id] = src_block_table.copy()
479
# When using a sliding window, blocks will be eventually reused.
vllm/core/block_manager_v2.py
+3
@@ -317,6 +317,9 @@ class BlockSpaceManagerV2(BlockSpaceManager):
317
computed_seq_block_ids) # type: ignore
318
319
def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
320
+ if parent_seq.seq_id not in self.block_tables:
321
+ # Parent sequence has either been freed or never existed.
322
+ return
323
src_block_table = self.block_tables[parent_seq.seq_id]
324
self.block_tables[child_seq.seq_id] = src_block_table.fork()
325
vllm/core/scheduler.py
+11
-2
@@ -256,6 +256,7 @@ class Scheduler:
256
scheduler_config: SchedulerConfig,
257
cache_config: CacheConfig,
258
lora_config: Optional[LoRAConfig],
259
+ pipeline_parallel_size: int = 1,
260
) -> None:
261
self.scheduler_config = scheduler_config
262
self.cache_config = cache_config
@@ -273,11 +274,19 @@ class Scheduler:
274
BlockSpaceManagerImpl = BlockSpaceManager.get_block_space_manager_class(
275
version)
276
277
+ num_gpu_blocks = cache_config.num_gpu_blocks
278
+ if num_gpu_blocks:
279
+ num_gpu_blocks //= pipeline_parallel_size
280
+
281
+ num_cpu_blocks = cache_config.num_cpu_blocks
282
+ if num_cpu_blocks:
283
+ num_cpu_blocks //= pipeline_parallel_size
284
+
285
# Create the block space manager.
286
self.block_manager = BlockSpaceManagerImpl(
287
block_size=self.cache_config.block_size,
279
- num_gpu_blocks=self.cache_config.num_gpu_blocks,
280
- num_cpu_blocks=self.cache_config.num_cpu_blocks,
288
+ num_gpu_blocks=num_gpu_blocks,
289
+ num_cpu_blocks=num_cpu_blocks,
290
sliding_window=self.cache_config.sliding_window,
291
enable_caching=self.cache_config.enable_prefix_caching)
292
vllm/distributed/parallel_state.py
+28
-22
@@ -416,7 +416,7 @@ class GroupCoordinator:
416
417
assert dst < self.world_size, f"Invalid dst rank ({dst})"
418
419
- assert dst != self.rank, (
419
+ assert dst != self.rank_in_group, (
420
"Invalid destination rank. Destination rank is the same "
421
"as the current rank.")
422
@@ -446,7 +446,7 @@ class GroupCoordinator:
446
447
assert src < self.world_size, f"Invalid src rank ({src})"
448
449
- assert src != self.rank, (
449
+ assert src != self.rank_in_group, (
450
"Invalid source rank. Source rank is the same as the current rank."
451
)
452
@@ -454,7 +454,7 @@ class GroupCoordinator:
454
455
# Receive object size
456
rank_size = torch.distributed.recv(size_tensor,
457
- src=src,
457
+ src=self.ranks[src],
458
group=self.cpu_group)
459
460
# Tensor to receive serialized objects into.
@@ -464,7 +464,7 @@ class GroupCoordinator:
464
device="cpu")
465
466
rank_object = torch.distributed.recv(object_tensor,
467
- src=src,
467
+ src=self.ranks[src],
468
group=self.cpu_group)
469
470
assert rank_object == rank_size, (
@@ -491,10 +491,9 @@ class GroupCoordinator:
491
group = self.device_group
492
metadata_group = self.cpu_group
493
assert src < self.world_size, f"Invalid src rank ({src})"
494
- src = self.ranks[src]
494
496
- rank = self.rank
497
- if rank == src:
495
+ rank_in_group = self.rank_in_group
496
+ if rank_in_group == src:
497
metadata_list: List[Tuple[Any, Any]] = []
498
assert isinstance(
499
tensor_dict,
@@ -512,13 +511,13 @@ class GroupCoordinator:
511
if tensor.is_cpu:
512
# use metadata_group for CPU tensors
513
handle = torch.distributed.broadcast(tensor,
515
- src=src,
514
+ src=self.ranks[src],
515
group=metadata_group,
516
async_op=True)
517
else:
518
# use group for GPU tensors
519
handle = torch.distributed.broadcast(tensor,
521
- src=src,
520
+ src=self.ranks[src],
521
group=group,
522
async_op=True)
523
async_handles.append(handle)
@@ -542,15 +541,16 @@ class GroupCoordinator:
541
# use metadata_group for CPU tensors
542
handle = torch.distributed.broadcast(
543
tensor,
545
- src=src,
544
+ src=self.ranks[src],
545
group=metadata_group,
546
async_op=True)
547
else:
548
# use group for GPU tensors
550
- handle = torch.distributed.broadcast(tensor,
551
- src=src,
552
- group=group,
553
- async_op=True)
549
+ handle = torch.distributed.broadcast(
550
+ tensor,
551
+ src=self.ranks[src],
552
+ group=group,
553
+ async_op=True)
554
async_handles.append(handle)
555
_update_nested_dict(tensor_dict, key, tensor)
556
else:
@@ -575,7 +575,7 @@ class GroupCoordinator:
575
metadata_group = self.cpu_group
576
577
if dst is None:
578
- dst = self.next_rank
578
+ dst = (self.rank_in_group + 1) % self.world_size
579
assert dst < self.world_size, f"Invalid dst rank ({dst})"
580
581
metadata_list: List[Tuple[Any, Any]] = []
@@ -593,10 +593,14 @@ class GroupCoordinator:
593
continue
594
if tensor.is_cpu:
595
# use metadata_group for CPU tensors
596
- torch.distributed.send(tensor, dst=dst, group=metadata_group)
596
+ torch.distributed.send(tensor,
597
+ dst=self.ranks[dst],
598
+ group=metadata_group)
599
else:
600
# use group for GPU tensors
599
- torch.distributed.send(tensor, dst=dst, group=group)
601
+ torch.distributed.send(tensor,
602
+ dst=self.ranks[dst],
603
+ group=group)
604
return None
605
606
def recv_tensor_dict(
@@ -614,7 +618,7 @@ class GroupCoordinator:
618
metadata_group = self.cpu_group
619
620
if src is None:
617
- src = self.prev_rank
621
+ src = (self.rank_in_group - 1) % self.world_size
622
assert src < self.world_size, f"Invalid src rank ({src})"
623
624
recv_metadata_list = self.recv_object(src=src)
@@ -631,11 +635,13 @@ class GroupCoordinator:
635
if tensor.is_cpu:
636
# use metadata_group for CPU tensors
637
torch.distributed.recv(tensor,
634
- src=src,
638
+ src=self.ranks[src],
639
group=metadata_group)
640
else:
641
# use group for GPU tensors
638
- torch.distributed.recv(tensor, src=src, group=group)
642
+ torch.distributed.recv(tensor,
643
+ src=self.ranks[src],
644
+ group=group)
645
_update_nested_dict(tensor_dict, key, tensor)
646
else:
647
_update_nested_dict(tensor_dict, key, value)
@@ -654,7 +660,7 @@ class GroupCoordinator:
660
"""Sends a tensor to the destination rank in a non-blocking way"""
661
"""NOTE: `dst` is the local rank of the destination rank."""
662
if dst is None:
657
- dst = self.next_rank
663
+ dst = (self.rank_in_group + 1) % self.world_size
664
665
pynccl_comm = self.pynccl_comm
666
if pynccl_comm is not None and not pynccl_comm.disabled:
@@ -669,7 +675,7 @@ class GroupCoordinator:
675
"""Receives a tensor from the src rank."""
676
"""NOTE: `src` is the local rank of the destination rank."""
677
if src is None:
672
- src = self.prev_rank
678
+ src = (self.rank_in_group - 1) % self.world_size
679
680
tensor = torch.empty(size, dtype=dtype, device=self.device)
681
pynccl_comm = self.pynccl_comm
vllm/distributed/utils.py
+10
-1
@@ -2,7 +2,7 @@
2
# Adapted from
3
# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/utils.py
4
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5
-from typing import Sequence
5
+from typing import Sequence, Tuple
6
7
import torch
8
@@ -46,3 +46,12 @@ def split_tensor_along_last_dim(
46
return tuple(chunk.contiguous() for chunk in tensor_list)
47
48
return tensor_list
49
+
50
+
51
+def get_pp_indices(num_hidden_layers: int, pp_rank: int,
52
+ pp_size: int) -> Tuple[int, int]:
53
+ layers_per_partition = divide(num_hidden_layers, pp_size)
54
+ start_layer = pp_rank * layers_per_partition
55
+ end_layer = start_layer + layers_per_partition
56
+
57
+ return (start_layer, end_layer)
vllm/engine/async_llm_engine.py
+63
-16
@@ -211,7 +211,8 @@ class _AsyncLLMEngine(LLMEngine):
211
"""Extension of LLMEngine to add async methods."""
212
213
async def step_async(
214
- self) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
214
+ self, virtual_engine: int
215
+ ) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
216
"""Performs one decoding iteration and returns newly generated results.
217
The workers are ran asynchronously if possible.
218
@@ -221,7 +222,8 @@ class _AsyncLLMEngine(LLMEngine):
222
and updates the scheduler with the model outputs. Finally, it decodes
223
the sequences and returns the newly generated results.
224
"""
224
- seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
225
+ seq_group_metadata_list, scheduler_outputs = self.scheduler[
226
+ virtual_engine].schedule()
227
228
if not scheduler_outputs.is_empty():
229
# Execute the model.
@@ -230,6 +232,7 @@ class _AsyncLLMEngine(LLMEngine):
232
blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
233
blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
234
blocks_to_copy=scheduler_outputs.blocks_to_copy,
235
+ virtual_engine=virtual_engine,
236
num_lookahead_slots=scheduler_outputs.num_lookahead_slots,
237
running_queue_size=scheduler_outputs.running_queue_size,
238
)
@@ -248,16 +251,12 @@ class _AsyncLLMEngine(LLMEngine):
251
# Tracing
252
self.do_tracing(scheduler_outputs)
253
251
- if not request_outputs:
252
- # Stop the execute model loop in parallel workers until there are
253
- # more requests to process. This avoids waiting indefinitely in
254
- # torch.distributed ops which may otherwise timeout, and unblocks
255
- # the RPC thread in the workers so that they can process any other
256
- # queued control plane messages, such as add/remove lora adapters.
257
- await self.model_executor.stop_remote_worker_execution_loop_async()
258
-
254
return request_outputs
255
256
+ async def stop_remote_worker_execution_loop_async(self) -> None:
257
+ """Stop the remote worker execution loop."""
258
+ await self.model_executor.stop_remote_worker_execution_loop_async()
259
+
260
async def process_model_inputs_async(
261
self,
262
request_id: str,
@@ -491,7 +490,8 @@ class AsyncLLMEngine:
490
# order of the arguments.
491
cache_config = kwargs["cache_config"]
492
parallel_config = kwargs["parallel_config"]
494
- if parallel_config.tensor_parallel_size == 1:
493
+ if (parallel_config.tensor_parallel_size == 1
494
+ and parallel_config.pipeline_parallel_size == 1):
495
num_gpus = cache_config.gpu_memory_utilization
496
else:
497
num_gpus = 1
@@ -499,7 +499,7 @@ class AsyncLLMEngine:
499
self._engine_class).remote
500
return engine_class(*args, **kwargs)
501
502
- async def engine_step(self) -> bool:
502
+ async def engine_step(self, virtual_engine: int) -> bool:
503
"""Kick the engine to process the waiting requests.
504
505
Returns True if there are in-progress requests."""
@@ -530,7 +530,7 @@ class AsyncLLMEngine:
530
if self.engine_use_ray:
531
request_outputs = await self.engine.step.remote() # type: ignore
532
else:
533
- request_outputs = await self.engine.step_async()
533
+ request_outputs = await self.engine.step_async(virtual_engine)
534
535
# Put the outputs into the corresponding streams.
536
for request_output in request_outputs:
@@ -546,18 +546,65 @@ class AsyncLLMEngine:
546
self.engine.abort_request(request_ids)
547
548
async def run_engine_loop(self):
549
- has_requests_in_progress = False
549
+ if self.engine_use_ray:
550
+ pipeline_parallel_size = 1 # type: ignore
551
+ else:
552
+ pipeline_parallel_size = \
553
+ self.engine.parallel_config.pipeline_parallel_size
554
+ has_requests_in_progress = [False] * pipeline_parallel_size
555
while True:
551
- if not has_requests_in_progress:
556
+ if not any(has_requests_in_progress):
557
logger.debug("Waiting for new requests...")
558
+ # Stop the execute model loop in parallel workers until there
559
+ # are more requests to process. This avoids waiting
560
+ # indefinitely in torch.distributed ops which may otherwise
561
+ # timeout, and unblocks the RPC thread in the workers so that
562
+ # they can process any other queued control plane messages,
563
+ # such as add/remove lora adapters.
564
+ if self.engine_use_ray:
565
+ await (self.engine.stop_remote_worker_execution_loop.
566
+ remote() # type: ignore
567
+ )
568
+ else:
569
+ await self.engine.stop_remote_worker_execution_loop_async()
570
await self._request_tracker.wait_for_new_requests()
571
logger.debug("Got new requests!")
572
+ requests_in_progress = [
573
+ asyncio.create_task(self.engine_step(ve))
574
+ for ve in range(pipeline_parallel_size)
575
+ ]
576
+ has_requests_in_progress = [True] * pipeline_parallel_size
577
578
# Abort if iteration takes too long due to unrecoverable errors
579
# (eg. NCCL timeouts).
580
try:
581
async with asyncio_timeout(ENGINE_ITERATION_TIMEOUT_S):
560
- has_requests_in_progress = await self.engine_step()
582
+ done, _ = await asyncio.wait(
583
+ requests_in_progress,
584
+ return_when=asyncio.FIRST_COMPLETED)
585
+ for _ in range(pipeline_parallel_size):
586
+ await asyncio.sleep(0)
587
+ for task in done:
588
+ result = task.result()
589
+ virtual_engine = requests_in_progress.index(task)
590
+ if self.engine_use_ray:
591
+ has_unfinished_requests = (
592
+ await (self.engine.
593
+ has_unfinished_requests_for_virtual_engine.
594
+ remote( # type: ignore
595
+ virtual_engine)))
596
+ else:
597
+ has_unfinished_requests = (
598
+ self.engine.
599
+ has_unfinished_requests_for_virtual_engine(
600
+ virtual_engine))
601
+ if result or has_unfinished_requests:
602
+ requests_in_progress[virtual_engine] = (
603
+ asyncio.create_task(
604
+ self.engine_step(virtual_engine)))
605
+ has_requests_in_progress[virtual_engine] = True
606
+ else:
607
+ has_requests_in_progress[virtual_engine] = False
608
except asyncio.TimeoutError as exc:
609
logger.error(
610
"Engine iteration timed out. This should never happen!")
vllm/engine/llm_engine.py
+50
-15
@@ -173,6 +173,7 @@ class LLMEngine:
173
"rope_scaling=%r, rope_theta=%r, tokenizer_revision=%s, "
174
"trust_remote_code=%s, dtype=%s, max_seq_len=%d, "
175
"download_dir=%r, load_format=%s, tensor_parallel_size=%d, "
176
+ "pipeline_parallel_size=%d, "
177
"disable_custom_all_reduce=%s, quantization=%s, "
178
"enforce_eager=%s, kv_cache_dtype=%s, "
179
"quantization_param_path=%s, device_config=%s, "
@@ -195,6 +196,7 @@ class LLMEngine:
196
load_config.download_dir,
197
load_config.load_format,
198
parallel_config.tensor_parallel_size,
199
+ parallel_config.pipeline_parallel_size,
200
parallel_config.disable_custom_all_reduce,
201
model_config.quantization,
202
model_config.enforce_eager,
@@ -296,7 +298,11 @@ class LLMEngine:
298
# Create the scheduler.
299
# NOTE: the cache_config here have been updated with the numbers of
300
# GPU and CPU blocks, which are profiled in the distributed executor.
299
- self.scheduler = Scheduler(scheduler_config, cache_config, lora_config)
301
+ self.scheduler = [
302
+ Scheduler(scheduler_config, cache_config, lora_config,
303
+ parallel_config.pipeline_parallel_size)
304
+ for _ in range(parallel_config.pipeline_parallel_size)
305
+ ]
306
307
# Metric Logging.
308
if self.log_stats:
@@ -513,8 +519,16 @@ class LLMEngine:
519
raise ValueError(
520
"Either SamplingParams or PoolingParams must be provided.")
521
516
- # Add the sequence group to the scheduler.
517
- self.scheduler.add_seq_group(seq_group)
522
+ # Add the sequence group to the scheduler with least unfinished seqs.
523
+ costs = [
524
+ scheduler.get_num_unfinished_seq_groups()
525
+ for scheduler in self.scheduler
526
+ ]
527
+ min_cost_scheduler = self.scheduler[costs.index(min(costs))]
528
+ min_cost_scheduler.add_seq_group(seq_group)
529
+
530
+ def stop_remote_worker_execution_loop(self) -> None:
531
+ self.model_executor.stop_remote_worker_execution_loop()
532
533
def process_model_inputs(
534
self,
@@ -684,7 +698,8 @@ class LLMEngine:
698
>>> # abort the request
699
>>> engine.abort_request(request_id)
700
"""
687
- self.scheduler.abort_seq_group(request_id)
701
+ for scheduler in self.scheduler:
702
+ scheduler.abort_seq_group(request_id)
703
704
def get_model_config(self) -> ModelConfig:
705
"""Gets the model configuration."""
@@ -696,11 +711,20 @@ class LLMEngine:
711
712
def get_num_unfinished_requests(self) -> int:
713
"""Gets the number of unfinished requests."""
699
- return self.scheduler.get_num_unfinished_seq_groups()
714
+ return sum(scheduler.get_num_unfinished_seq_groups()
715
+ for scheduler in self.scheduler)
716
717
def has_unfinished_requests(self) -> bool:
718
"""Returns True if there are unfinished requests."""
703
- return self.scheduler.has_unfinished_seqs()
719
+ return any(scheduler.has_unfinished_seqs()
720
+ for scheduler in self.scheduler)
721
+
722
+ def has_unfinished_requests_for_virtual_engine(
723
+ self, virtual_engine: int) -> bool:
724
+ """
725
+ Returns True if there are unfinished requests for the virtual engine.
726
+ """
727
+ return self.scheduler[virtual_engine].has_unfinished_seqs()
728
729
def _process_sequence_group_outputs(
730
self,
@@ -749,7 +773,8 @@ class LLMEngine:
773
self.output_processor.process_outputs(seq_group, outputs)
774
775
# Free the finished sequence groups.
752
- self.scheduler.free_finished_seq_groups()
776
+ for scheduler in self.scheduler:
777
+ scheduler.free_finished_seq_groups()
778
779
# Create the outputs.
780
request_outputs: List[Union[RequestOutput,
@@ -815,7 +840,12 @@ class LLMEngine:
840
>>> if not (engine.has_unfinished_requests() or example_inputs):
841
>>> break
842
"""
818
- seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
843
+ if self.parallel_config.pipeline_parallel_size > 1:
844
+ raise NotImplementedError(
845
+ "Pipeline parallelism is only supported through AsyncLLMEngine "
846
+ "as performance will be severely degraded otherwise.")
847
+ seq_group_metadata_list, scheduler_outputs = self.scheduler[
848
+ 0].schedule()
849
850
if not scheduler_outputs.is_empty():
851
execute_model_req = ExecuteModelRequest(
@@ -886,23 +916,28 @@ class LLMEngine:
916
917
# System State
918
# Scheduler State
889
- num_running_sys = len(self.scheduler.running)
890
- num_swapped_sys = len(self.scheduler.swapped)
891
- num_waiting_sys = len(self.scheduler.waiting)
919
+ num_running_sys = sum(
920
+ len(scheduler.running) for scheduler in self.scheduler)
921
+ num_swapped_sys = sum(
922
+ len(scheduler.swapped) for scheduler in self.scheduler)
923
+ num_waiting_sys = sum(
924
+ len(scheduler.waiting) for scheduler in self.scheduler)
925
926
# KV Cache Usage in %
927
num_total_gpu = self.cache_config.num_gpu_blocks
928
gpu_cache_usage_sys = 0.
929
if num_total_gpu is not None:
897
- num_free_gpu = self.scheduler.block_manager.get_num_free_gpu_blocks(
898
- )
930
+ num_free_gpu = sum(
931
+ scheduler.block_manager.get_num_free_gpu_blocks()
932
+ for scheduler in self.scheduler)
933
gpu_cache_usage_sys = 1.0 - (num_free_gpu / num_total_gpu)
934
935
num_total_cpu = self.cache_config.num_cpu_blocks
936
cpu_cache_usage_sys = 0.
937
if num_total_cpu is not None and num_total_cpu > 0:
904
- num_free_cpu = self.scheduler.block_manager.get_num_free_cpu_blocks(
905
- )
938
+ num_free_cpu = sum(
939
+ scheduler.block_manager.get_num_free_cpu_blocks()
940
+ for scheduler in self.scheduler)
941
cpu_cache_usage_sys = 1.0 - (num_free_cpu / num_total_cpu)
942
943
# Iteration stats
vllm/engine/output_processor/interfaces.py
+1
-1
@@ -27,7 +27,7 @@ class SequenceGroupOutputProcessor(ABC):
27
def create_output_processor(
28
scheduler_config: SchedulerConfig,
29
detokenizer: Detokenizer,
30
- scheduler: Scheduler,
30
+ scheduler: List[Scheduler],
31
seq_counter: Counter,
32
get_tokenizer_for_seq: Callable[[Sequence], PreTrainedTokenizer],
33
stop_checker: "StopChecker",
vllm/engine/output_processor/multi_step.py
+3
-2
@@ -34,7 +34,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
34
def __init__(
35
self,
36
detokenizer: Detokenizer,
37
- scheduler: Scheduler,
37
+ scheduler: List[Scheduler],
38
seq_counter: Counter,
39
get_tokenizer_for_seq: Callable[[Sequence], PreTrainedTokenizer],
40
stop_checker: StopChecker,
@@ -141,4 +141,5 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
141
break
142
143
if seq.is_finished():
144
- self.scheduler.free_seq(seq)
144
+ for scheduler in self.scheduler:
145
+ scheduler.free_seq(seq)
vllm/engine/output_processor/single_step.py
+13
-7
@@ -33,7 +33,7 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
33
self,
34
scheduler_config: SchedulerConfig,
35
detokenizer: Detokenizer,
36
- scheduler: Scheduler,
36
+ scheduler: List[Scheduler],
37
seq_counter: Counter,
38
stop_checker: StopChecker,
39
):
@@ -95,7 +95,8 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
95
# not be used in the future iterations.
96
parent.status = SequenceStatus.FINISHED_ABORTED
97
seq_group.remove(parent.seq_id)
98
- self.scheduler.free_seq(parent)
98
+ for scheduler in self.scheduler:
99
+ scheduler.free_seq(parent)
100
continue
101
# Fork the parent sequence if there are multiple child samples.
102
for child_sample in child_samples[:-1]:
@@ -133,7 +134,8 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
134
if seq is not parent:
135
seq_group.add(seq)
136
if not seq.is_finished():
136
- self.scheduler.fork_seq(parent, seq)
137
+ for scheduler in self.scheduler:
138
+ scheduler.fork_seq(parent, seq)
139
140
# Free the finished and selected parent sequences' memory in block
141
# manager. Keep them in the sequence group as candidate output.
@@ -141,7 +143,8 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
143
# old sequences.
144
for seq, parent in child_seqs:
145
if seq is parent and seq.is_finished():
144
- self.scheduler.free_seq(seq)
146
+ for scheduler in self.scheduler:
147
+ scheduler.free_seq(seq)
148
return
149
150
# Beam search case
@@ -226,13 +229,15 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
229
if seq is not parent:
230
seq_group.add(seq)
231
if not seq.is_finished():
229
- self.scheduler.fork_seq(parent, seq)
232
+ for scheduler in self.scheduler:
233
+ scheduler.fork_seq(parent, seq)
234
235
# Free the finished and selected parent sequences' memory in block
236
# manager. Keep them in the sequence group as candidate output.
237
for seq, parent in selected_child_seqs:
238
if seq is parent and seq.is_finished():
235
- self.scheduler.free_seq(seq)
239
+ for scheduler in self.scheduler:
240
+ scheduler.free_seq(seq)
241
242
# Remove the unselected parent sequences from the sequence group and
243
# free their memory in block manager.
@@ -241,7 +246,8 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
246
# Remove the parent sequence if it is not selected for next
247
# iteration
248
seq_group.remove(seq.seq_id)
244
- self.scheduler.free_seq(seq)
249
+ for scheduler in self.scheduler:
250
+ scheduler.free_seq(seq)
251
252
def _check_beam_search_early_stopping(
253
self,
vllm/executor/distributed_gpu_executor.py
+6
-6
@@ -69,7 +69,7 @@ class DistributedGPUExecutor(GPUExecutor):
69
if self.parallel_worker_tasks is None:
70
self.parallel_worker_tasks = self._run_workers(
71
"start_worker_execution_loop",
72
- async_run_remote_workers_only=True,
72
+ async_run_tensor_parallel_workers_only=True,
73
**self.extra_execute_model_run_workers_kwargs)
74
75
# Only the driver worker returns the sampling results.
@@ -138,17 +138,17 @@ class DistributedGPUExecutor(GPUExecutor):
138
self,
139
method: str,
140
*args,
141
- async_run_remote_workers_only: bool = False,
141
+ async_run_tensor_parallel_workers_only: bool = False,
142
max_concurrent_workers: Optional[int] = None,
143
**kwargs,
144
) -> Any:
145
"""Runs the given method on all workers.
146
147
Args:
148
- async_run_remote_workers_only: If True the method will be run only
149
- in the remote workers, not the driver worker. It will also be
150
- run asynchronously and return a list of futures rather than
151
- blocking on the results.
148
+ async_run_tensor_parallel_workers_only: If True the method will be
149
+ run only in the remote TP workers, not the driver worker.
150
+ It will also be run asynchronously and return a list of futures
151
+ rather than blocking on the results.
152
"""
153
raise NotImplementedError
154
vllm/executor/executor_base.py
+25
@@ -1,3 +1,4 @@
1
+import asyncio
2
from abc import ABC, abstractmethod
3
from typing import List, Optional, Set, Tuple
4
@@ -110,6 +111,30 @@ class ExecutorBase(ABC):
111
112
class ExecutorAsyncBase(ExecutorBase):
113
114
+ def __init__(
115
+ self,
116
+ model_config: ModelConfig,
117
+ cache_config: CacheConfig,
118
+ parallel_config: ParallelConfig,
119
+ scheduler_config: SchedulerConfig,
120
+ device_config: DeviceConfig,
121
+ load_config: LoadConfig,
122
+ lora_config: Optional[LoRAConfig],
123
+ vision_language_config: Optional[VisionLanguageConfig],
124
+ speculative_config: Optional[SpeculativeConfig],
125
+ ) -> None:
126
+ # This locks each pipeline parallel stage so multiple virtual engines
127
+ # can't execute on the same stage at the same time
128
+ self.pp_locks = [
129
+ asyncio.Lock()
130
+ for _ in range(parallel_config.pipeline_parallel_size)
131
+ ]
132
+
133
+ super().__init__(model_config, cache_config, parallel_config,
134
+ scheduler_config, device_config, load_config,
135
+ lora_config, vision_language_config,
136
+ speculative_config)
137
+
138
@abstractmethod
139
async def execute_model_async(
140
self,
vllm/executor/gpu_executor.py
+2
-1
@@ -45,7 +45,8 @@ class GPUExecutor(ExecutorBase):
45
lora_config=self.lora_config,
46
vision_language_config=self.vision_language_config,
47
speculative_config=self.speculative_config,
48
- is_driver_worker=rank == 0,
48
+ is_driver_worker=(not self.parallel_config)
49
+ or (rank % self.parallel_config.tensor_parallel_size == 0),
50
)
51
52
def _create_worker(self,
vllm/executor/multiproc_gpu_executor.py
+6
-6
@@ -91,17 +91,17 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
91
self,
92
method: str,
93
*args,
94
- async_run_remote_workers_only: bool = False,
94
+ async_run_tensor_parallel_workers_only: bool = False,
95
max_concurrent_workers: Optional[int] = None,
96
**kwargs,
97
) -> Any:
98
"""Runs the given method on all workers.
99
100
Args:
101
- async_run_remote_workers_only: If True the method will be run only
102
- in the remote workers, not the driver worker. It will also be
103
- run asynchronously and return a list of futures rather than
104
- blocking on the results.
101
+ async_run_tensor_parallel_workers_only: If True the method will be
102
+ run only in the remote TP workers, not the driver worker.
103
+ It will also be run asynchronously and return a list of futures
104
+ rather than blocking on the results.
105
"""
106
107
if max_concurrent_workers:
@@ -114,7 +114,7 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
114
for worker in self.workers
115
]
116
117
- if async_run_remote_workers_only:
117
+ if async_run_tensor_parallel_workers_only:
118
# Just return futures
119
return worker_outputs
120
vllm/executor/ray_gpu_executor.py
+59
-12
@@ -62,7 +62,8 @@ class RayGPUExecutor(DistributedGPUExecutor):
62
63
def _init_workers_ray(self, placement_group: "PlacementGroup",
64
**ray_remote_kwargs):
65
- if self.parallel_config.tensor_parallel_size == 1:
65
+ if (self.parallel_config.tensor_parallel_size == 1
66
+ and self.parallel_config.pipeline_parallel_size == 1):
67
# For single GPU case, we use a ray worker with constrained memory.
68
num_gpus = self.cache_config.gpu_memory_utilization
69
else:
@@ -189,6 +190,26 @@ class RayGPUExecutor(DistributedGPUExecutor):
190
max_concurrent_workers=self.parallel_config.
191
max_parallel_loading_workers)
192
193
+ # This is the list of workers that are rank 0 of each TP group EXCEPT
194
+ # global rank 0. These are the workers that will broadcast to the
195
+ # rest of the workers.
196
+ self.tp_driver_workers: List[RayWorkerWrapper] = []
197
+ # This is the list of workers that are not drivers and not the first
198
+ # worker in a TP group. These are the workers that will be
199
+ # broadcasted to.
200
+ self.non_driver_workers: List[RayWorkerWrapper] = []
201
+
202
+ for pp_rank in range(self.parallel_config.pipeline_parallel_size):
203
+ for tp_rank in range(self.parallel_config.tensor_parallel_size):
204
+ rank = (pp_rank *
205
+ self.parallel_config.tensor_parallel_size) + tp_rank
206
+ if rank == 0:
207
+ pass
208
+ elif rank % self.parallel_config.tensor_parallel_size == 0:
209
+ self.tp_driver_workers.append(self.workers[rank - 1])
210
+ else:
211
+ self.non_driver_workers.append(self.workers[rank - 1])
212
+
213
def _driver_execute_model(
214
self, execute_model_req: Optional[ExecuteModelRequest]
215
) -> Optional[List[SamplerOutput]]:
@@ -204,7 +225,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
225
self,
226
method: str,
227
*args,
207
- async_run_remote_workers_only: bool = False,
228
+ async_run_tensor_parallel_workers_only: bool = False,
229
all_args: Optional[List[Tuple[Any, ...]]] = None,
230
all_kwargs: Optional[List[Dict[str, Any]]] = None,
231
use_dummy_driver: bool = False,
@@ -215,10 +236,11 @@ class RayGPUExecutor(DistributedGPUExecutor):
236
"""Runs the given method on all workers. Can be used in the following
237
ways:
238
218
- - async_run_remote_workers_only: If True the method will be run only
219
- in the remote workers, not the driver worker. It will also be
220
- run asynchronously and return a list of futures rather than blocking
221
- on the results.
239
+ Args:
240
+ - async_run_tensor_parallel_workers_only: If True the method will be
241
+ run only in the remote TP workers, not the driver worker.
242
+ It will also be run asynchronously and return a list of futures
243
+ rather than blocking on the results.
244
- args/kwargs: All workers share the same args/kwargs
245
- all_args/all_kwargs: args/kwargs for each worker are specified
246
individually
@@ -228,7 +250,9 @@ class RayGPUExecutor(DistributedGPUExecutor):
250
raise NotImplementedError(
251
"max_concurrent_workers is not supported yet.")
252
231
- count = len(self.workers)
253
+ count = len(self.workers) if not \
254
+ async_run_tensor_parallel_workers_only \
255
+ else len(self.non_driver_workers)
256
all_worker_args = repeat(args, count) if all_args is None \
257
else islice(all_args, 1, None)
258
all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
@@ -242,14 +266,17 @@ class RayGPUExecutor(DistributedGPUExecutor):
266
ray_worker_outputs = []
267
else:
268
# Start the ray workers first.
269
+ ray_workers = self.workers
270
+ if async_run_tensor_parallel_workers_only:
271
+ ray_workers = self.non_driver_workers
272
ray_worker_outputs = [
273
worker.execute_method.remote(method, *worker_args,
274
**worker_kwargs)
275
for (worker, worker_args, worker_kwargs
249
- ) in zip(self.workers, all_worker_args, all_worker_kwargs)
276
+ ) in zip(ray_workers, all_worker_args, all_worker_kwargs)
277
]
278
252
- if async_run_remote_workers_only:
279
+ if async_run_tensor_parallel_workers_only:
280
# Just return futures
281
return ray_worker_outputs
282
@@ -319,12 +346,32 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
346
self,
347
execute_model_req: Optional[ExecuteModelRequest] = None
348
) -> List[SamplerOutput]:
322
- return await self.driver_exec_method("execute_model",
323
- execute_model_req)
349
+
350
+ async def _run_task_with_lock(task, lock, *args, **kwargs):
351
+ async with lock:
352
+ return await task(*args, **kwargs)
353
+
354
+ tasks = []
355
+ tasks.append(
356
+ asyncio.create_task(
357
+ _run_task_with_lock(self.driver_exec_method, self.pp_locks[0],
358
+ "execute_model", execute_model_req)))
359
+ for pp_rank, driver_worker in enumerate(self.tp_driver_workers,
360
+ start=1):
361
+ tasks.append(
362
+ asyncio.create_task(
363
+ _run_task_with_lock(driver_worker.execute_method.remote,
364
+ self.pp_locks[pp_rank],
365
+ "execute_model", execute_model_req)))
366
+
367
+ results = await asyncio.gather(*tasks)
368
+
369
+ # Only the last PP stage has the final results.
370
+ return results[-1]
371
372
async def _start_worker_execution_loop(self):
373
coros = [
374
worker.execute_method.remote("start_worker_execution_loop")
328
- for worker in self.workers
375
+ for worker in self.non_driver_workers
376
]
377
return await asyncio.gather(*coros)
vllm/model_executor/models/arctic.py
+2
-1
@@ -29,7 +29,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
29
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
30
from vllm.model_executor.sampling_metadata import SamplingMetadata
31
from vllm.model_executor.utils import set_weight_attrs
32
-from vllm.sequence import SamplerOutput
32
+from vllm.sequence import IntermediateTensors, SamplerOutput
33
from vllm.transformers_utils.configs.arctic import ArcticConfig
34
35
logger = init_logger(__name__)
@@ -426,6 +426,7 @@ class ArcticForCausalLM(nn.Module):
426
positions: torch.Tensor,
427
kv_caches: List[torch.Tensor],
428
attn_metadata: AttentionMetadata,
429
+ intermediate_tensors: Optional[IntermediateTensors] = None,
430
) -> torch.Tensor:
431
hidden_states = self.model(input_ids, positions, kv_caches,
432
attn_metadata)
vllm/model_executor/models/baichuan.py
+2
-1
@@ -43,7 +43,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
43
ParallelLMHead, VocabParallelEmbedding)
44
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45
from vllm.model_executor.sampling_metadata import SamplingMetadata
46
-from vllm.sequence import SamplerOutput
46
+from vllm.sequence import IntermediateTensors, SamplerOutput
47
48
from .interfaces import SupportsLoRA
49
@@ -338,6 +338,7 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA):
338
positions: torch.Tensor,
339
kv_caches: List[torch.Tensor],
340
attn_metadata: AttentionMetadata,
341
+ intermediate_tensors: Optional[IntermediateTensors] = None,
342
) -> torch.Tensor:
343
hidden_states = self.model(input_ids, positions, kv_caches,
344
attn_metadata)
vllm/model_executor/models/bloom.py
+2
-1
@@ -39,7 +39,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
39
VocabParallelEmbedding)
40
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41
from vllm.model_executor.sampling_metadata import SamplingMetadata
42
-from vllm.sequence import SamplerOutput
42
+from vllm.sequence import IntermediateTensors, SamplerOutput
43
44
45
def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
@@ -286,6 +286,7 @@ class BloomForCausalLM(nn.Module):
286
positions: torch.Tensor,
287
kv_caches: List[torch.Tensor],
288
attn_metadata: AttentionMetadata,
289
+ intermediate_tensors: Optional[IntermediateTensors] = None,
290
) -> torch.Tensor:
291
hidden_states = self.transformer(input_ids, positions, kv_caches,
292
attn_metadata)
vllm/model_executor/models/chatglm.py
+2
-1
@@ -25,7 +25,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
25
ParallelLMHead, VocabParallelEmbedding)
26
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27
from vllm.model_executor.sampling_metadata import SamplingMetadata
28
-from vllm.sequence import SamplerOutput
28
+from vllm.sequence import IntermediateTensors, SamplerOutput
29
from vllm.transformers_utils.configs import ChatGLMConfig
30
31
from .interfaces import SupportsLoRA
@@ -365,6 +365,7 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA):
365
positions: torch.Tensor,
366
kv_caches: List[torch.Tensor],
367
attn_metadata: AttentionMetadata,
368
+ intermediate_tensors: Optional[IntermediateTensors] = None,
369
) -> torch.Tensor:
370
hidden_states = self.transformer(input_ids, positions, kv_caches,
371
attn_metadata)
vllm/model_executor/models/commandr.py
+2
-1
@@ -46,7 +46,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
46
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
47
from vllm.model_executor.sampling_metadata import SamplingMetadata
48
from vllm.model_executor.utils import set_weight_attrs
49
-from vllm.sequence import SamplerOutput
49
+from vllm.sequence import IntermediateTensors, SamplerOutput
50
51
52
@torch.compile
@@ -353,6 +353,7 @@ class CohereForCausalLM(nn.Module):
353
positions: torch.Tensor,
354
kv_caches: List[torch.Tensor],
355
attn_metadata: AttentionMetadata,
356
+ intermediate_tensors: Optional[IntermediateTensors] = None,
357
) -> torch.Tensor:
358
hidden_states = self.model(input_ids, positions, kv_caches,
359
attn_metadata)
vllm/model_executor/models/dbrx.py
+2
-1
@@ -23,7 +23,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
23
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24
from vllm.model_executor.sampling_metadata import SamplingMetadata
25
from vllm.model_executor.utils import set_weight_attrs
26
-from vllm.sequence import SamplerOutput
26
+from vllm.sequence import IntermediateTensors, SamplerOutput
27
from vllm.transformers_utils.configs.dbrx import DbrxConfig
28
29
@@ -381,6 +381,7 @@ class DbrxForCausalLM(nn.Module):
381
positions: torch.Tensor,
382
kv_caches: List[torch.Tensor],
383
attn_metadata: AttentionMetadata,
384
+ intermediate_tensors: Optional[IntermediateTensors] = None,
385
) -> torch.Tensor:
386
hidden_states = self.transformer(input_ids, positions, kv_caches,
387
attn_metadata)
vllm/model_executor/models/deepseek.py
+2
-1
@@ -48,7 +48,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
48
ParallelLMHead, VocabParallelEmbedding)
49
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
50
from vllm.model_executor.sampling_metadata import SamplingMetadata
51
-from vllm.sequence import SamplerOutput
51
+from vllm.sequence import IntermediateTensors, SamplerOutput
52
53
54
class DeepseekMLP(nn.Module):
@@ -387,6 +387,7 @@ class DeepseekForCausalLM(nn.Module):
387
positions: torch.Tensor,
388
kv_caches: List[torch.Tensor],
389
attn_metadata: AttentionMetadata,
390
+ intermediate_tensors: Optional[IntermediateTensors] = None,
391
) -> torch.Tensor:
392
hidden_states = self.model(input_ids, positions, kv_caches,
393
attn_metadata)
vllm/model_executor/models/deepseek_v2.py
+2
-1
@@ -48,7 +48,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
48
ParallelLMHead, VocabParallelEmbedding)
49
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
50
from vllm.model_executor.sampling_metadata import SamplingMetadata
51
-from vllm.sequence import SamplerOutput
51
+from vllm.sequence import IntermediateTensors, SamplerOutput
52
53
54
class DeepseekV2MLP(nn.Module):
@@ -475,6 +475,7 @@ class DeepseekV2ForCausalLM(nn.Module):
475
positions: torch.Tensor,
476
kv_caches: List[torch.Tensor],
477
attn_metadata: AttentionMetadata,
478
+ intermediate_tensors: Optional[IntermediateTensors] = None,
479
) -> torch.Tensor:
480
hidden_states = self.model(input_ids, positions, kv_caches,
481
attn_metadata)
vllm/model_executor/models/falcon.py
+2
-1
@@ -44,7 +44,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
44
ParallelLMHead, VocabParallelEmbedding)
45
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
46
from vllm.model_executor.sampling_metadata import SamplingMetadata
47
-from vllm.sequence import SamplerOutput
47
+from vllm.sequence import IntermediateTensors, SamplerOutput
48
from vllm.transformers_utils.configs import RWConfig
49
50
FalconConfig = Union[HF_FalconConfig, RWConfig]
@@ -410,6 +410,7 @@ class FalconForCausalLM(nn.Module):
410
positions: torch.Tensor,
411
kv_caches: List[torch.Tensor],
412
attn_metadata: AttentionMetadata,
413
+ intermediate_tensors: Optional[IntermediateTensors] = None,
414
) -> torch.Tensor:
415
hidden_states = self.transformer(
416
input_ids,
vllm/model_executor/models/gemma.py
+2
-1
@@ -39,7 +39,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
39
VocabParallelEmbedding)
40
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41
from vllm.model_executor.sampling_metadata import SamplingMetadata
42
-from vllm.sequence import SamplerOutput
42
+from vllm.sequence import IntermediateTensors, SamplerOutput
43
44
from .interfaces import SupportsLoRA
45
@@ -339,6 +339,7 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA):
339
positions: torch.Tensor,
340
kv_caches: List[torch.Tensor],
341
attn_metadata: AttentionMetadata,
342
+ intermediate_tensors: Optional[IntermediateTensors] = None,
343
) -> torch.Tensor:
344
hidden_states = self.model(input_ids, positions, kv_caches,
345
attn_metadata)
vllm/model_executor/models/gemma2.py
+2
-1
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
37
VocabParallelEmbedding)
38
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
39
from vllm.model_executor.sampling_metadata import SamplingMetadata
40
-from vllm.sequence import SamplerOutput
40
+from vllm.sequence import IntermediateTensors, SamplerOutput
41
from vllm.utils import print_warning_once
42
43
from .interfaces import SupportsLoRA
@@ -338,6 +338,7 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA):
338
positions: torch.Tensor,
339
kv_caches: List[torch.Tensor],
340
attn_metadata: AttentionMetadata,
341
+ intermediate_tensors: Optional[IntermediateTensors] = None,
342
) -> torch.Tensor:
343
hidden_states = self.model(input_ids, positions, kv_caches,
344
attn_metadata)
vllm/model_executor/models/gpt2.py
+61
-27
@@ -17,7 +17,7 @@
17
# See the License for the specific language governing permissions and
18
# limitations under the License.
19
"""Inference-only GPT-2 model compatible with HuggingFace weights."""
20
-from typing import Iterable, List, Optional, Tuple
20
+from typing import Iterable, List, Optional, Tuple, Union
21
22
import torch
23
from torch import nn
@@ -25,7 +25,9 @@ from transformers import GPT2Config
25
26
from vllm.attention import Attention, AttentionMetadata
27
from vllm.config import CacheConfig
28
-from vllm.distributed import get_tensor_model_parallel_world_size
28
+from vllm.distributed.parallel_state import (
29
+ get_pp_group, get_tensor_model_parallel_world_size)
30
+from vllm.distributed.utils import get_pp_indices
31
from vllm.model_executor.layers.activation import get_act_fn
32
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33
QKVParallelLinear,
@@ -38,7 +40,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
40
VocabParallelEmbedding)
41
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
42
from vllm.model_executor.sampling_metadata import SamplingMetadata
41
-from vllm.sequence import SamplerOutput
43
+from vllm.sequence import IntermediateTensors, SamplerOutput
44
45
46
class GPT2Attention(nn.Module):
@@ -181,10 +183,18 @@ class GPT2Model(nn.Module):
183
self.embed_dim = config.hidden_size
184
self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
185
self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
184
- self.h = nn.ModuleList([
185
- GPT2Block(config, cache_config, quant_config)
186
- for _ in range(config.num_hidden_layers)
187
- ])
186
+ self.start_layer, self.end_layer = get_pp_indices(
187
+ config.num_hidden_layers,
188
+ get_pp_group().rank_in_group,
189
+ get_pp_group().world_size)
190
+ self.h = nn.ModuleList(
191
+ [nn.Identity() for _ in range(self.start_layer)] + [
192
+ GPT2Block(config, cache_config, quant_config)
193
+ for _ in range(self.start_layer, self.end_layer)
194
+ ] + [
195
+ nn.Identity()
196
+ for _ in range(self.end_layer, config.num_hidden_layers)
197
+ ])
198
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
199
200
def forward(
@@ -193,14 +203,24 @@ class GPT2Model(nn.Module):
203
position_ids: torch.Tensor,
204
kv_caches: List[torch.Tensor],
205
attn_metadata: AttentionMetadata,
196
- ) -> torch.Tensor:
197
- inputs_embeds = self.wte(input_ids)
198
- position_embeds = self.wpe(position_ids)
199
- hidden_states = inputs_embeds + position_embeds
206
+ intermediate_tensors: Optional[IntermediateTensors],
207
+ ) -> Union[torch.Tensor, IntermediateTensors]:
208
+ if get_pp_group().is_first_rank:
209
+ inputs_embeds = self.wte(input_ids)
210
+ position_embeds = self.wpe(position_ids)
211
+ hidden_states = inputs_embeds + position_embeds
212
+ else:
213
+ assert intermediate_tensors is not None
214
+ hidden_states = intermediate_tensors["hidden_states"]
215
201
- for i in range(len(self.h)):
216
+ for i in range(self.start_layer, self.end_layer):
217
layer = self.h[i]
203
- hidden_states = layer(hidden_states, kv_caches[i], attn_metadata)
218
+ hidden_states = layer(hidden_states,
219
+ kv_caches[i - self.start_layer],
220
+ attn_metadata)
221
+
222
+ if not get_pp_group().is_last_rank:
223
+ return IntermediateTensors({"hidden_states": hidden_states})
224
225
hidden_states = self.ln_f(hidden_states)
226
return hidden_states
@@ -228,9 +248,10 @@ class GPT2LMHeadModel(nn.Module):
248
positions: torch.Tensor,
249
kv_caches: List[torch.Tensor],
250
attn_metadata: AttentionMetadata,
251
+ intermediate_tensors: Optional[IntermediateTensors] = None,
252
) -> torch.Tensor:
253
hidden_states = self.transformer(input_ids, positions, kv_caches,
233
- attn_metadata)
254
+ attn_metadata, intermediate_tensors)
255
return hidden_states
256
257
def compute_logits(self, hidden_states: torch.Tensor,
@@ -247,6 +268,16 @@ class GPT2LMHeadModel(nn.Module):
268
next_tokens = self.sampler(logits, sampling_metadata)
269
return next_tokens
270
271
+ def make_empty_intermediate_tensors(
272
+ self, batch_size: int, dtype: torch.dtype,
273
+ device: torch.device) -> IntermediateTensors:
274
+ return IntermediateTensors({
275
+ "hidden_states":
276
+ torch.zeros((batch_size, self.config.hidden_size),
277
+ dtype=dtype,
278
+ device=device),
279
+ })
280
+
281
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
282
params_dict = dict(self.named_parameters(remove_duplicate=False))
283
for name, loaded_weight in weights:
@@ -260,16 +291,19 @@ class GPT2LMHeadModel(nn.Module):
291
continue
292
if not name.startswith("transformer."):
293
name = "transformer." + name
263
- param = params_dict[name]
264
- # The HF's GPT-2 implementation uses Conv1D instead of Linear.
265
- # Because of this, we need to transpose the weights.
266
- # Note(zhuohan): the logic below might break quantized models.
267
- for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]:
268
- if conv1d_weight_name not in name:
269
- continue
270
- if not name.endswith(".weight"):
271
- continue
272
- loaded_weight = loaded_weight.t()
273
- weight_loader = getattr(param, "weight_loader",
274
- default_weight_loader)
275
- weight_loader(param, loaded_weight)
294
+ try:
295
+ param = params_dict[name]
296
+ # The HF's GPT-2 implementation uses Conv1D instead of Linear.
297
+ # Because of this, we need to transpose the weights.
298
+ # Note(zhuohan): the logic below might break quantized models.
299
+ for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]:
300
+ if conv1d_weight_name not in name:
301
+ continue
302
+ if not name.endswith(".weight"):
303
+ continue
304
+ loaded_weight = loaded_weight.t()
305
+ weight_loader = getattr(param, "weight_loader",
306
+ default_weight_loader)
307
+ weight_loader(param, loaded_weight)
308
+ except KeyError:
309
+ continue
vllm/model_executor/models/gpt_bigcode.py
+2
-1
@@ -39,7 +39,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
39
VocabParallelEmbedding)
40
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41
from vllm.model_executor.sampling_metadata import SamplingMetadata
42
-from vllm.sequence import SamplerOutput
42
+from vllm.sequence import IntermediateTensors, SamplerOutput
43
44
from .interfaces import SupportsLoRA
45
@@ -273,6 +273,7 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA):
273
positions: torch.Tensor,
274
kv_caches: List[torch.Tensor],
275
attn_metadata: AttentionMetadata,
276
+ intermediate_tensors: Optional[IntermediateTensors] = None,
277
) -> torch.Tensor:
278
hidden_states = self.transformer(input_ids, positions, kv_caches,
279
attn_metadata)
vllm/model_executor/models/gpt_j.py
+2
-1
@@ -38,7 +38,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
38
ParallelLMHead, VocabParallelEmbedding)
39
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40
from vllm.model_executor.sampling_metadata import SamplingMetadata
41
-from vllm.sequence import SamplerOutput
41
+from vllm.sequence import IntermediateTensors, SamplerOutput
42
43
44
class GPTJAttention(nn.Module):
@@ -239,6 +239,7 @@ class GPTJForCausalLM(nn.Module):
239
positions: torch.Tensor,
240
kv_caches: List[torch.Tensor],
241
attn_metadata: AttentionMetadata,
242
+ intermediate_tensors: Optional[IntermediateTensors] = None,
243
) -> torch.Tensor:
244
hidden_states = self.transformer(input_ids, positions, kv_caches,
245
attn_metadata)
vllm/model_executor/models/gpt_neox.py
+2
-1
@@ -38,7 +38,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
38
ParallelLMHead, VocabParallelEmbedding)
39
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40
from vllm.model_executor.sampling_metadata import SamplingMetadata
41
-from vllm.sequence import SamplerOutput
41
+from vllm.sequence import IntermediateTensors, SamplerOutput
42
43
44
class GPTNeoXAttention(nn.Module):
@@ -251,6 +251,7 @@ class GPTNeoXForCausalLM(nn.Module):
251
positions: torch.Tensor,
252
kv_caches: List[torch.Tensor],
253
attn_metadata: AttentionMetadata,
254
+ intermediate_tensors: Optional[IntermediateTensors] = None,
255
) -> torch.Tensor:
256
hidden_states = self.gpt_neox(input_ids, positions, kv_caches,
257
attn_metadata)
vllm/model_executor/models/internlm2.py
+2
-1
@@ -22,7 +22,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
22
ParallelLMHead, VocabParallelEmbedding)
23
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24
from vllm.model_executor.sampling_metadata import SamplingMetadata
25
-from vllm.sequence import SamplerOutput
25
+from vllm.sequence import IntermediateTensors, SamplerOutput
26
27
28
class InternLM2MLP(nn.Module):
@@ -263,6 +263,7 @@ class InternLM2ForCausalLM(nn.Module):
263
positions: torch.Tensor,
264
kv_caches: List[torch.Tensor],
265
attn_metadata: AttentionMetadata,
266
+ intermediate_tensors: IntermediateTensors,
267
) -> torch.Tensor:
268
hidden_states = self.model(input_ids, positions, kv_caches,
269
attn_metadata)
vllm/model_executor/models/jais.py
+2
-1
@@ -40,7 +40,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
40
VocabParallelEmbedding)
41
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
42
from vllm.model_executor.sampling_metadata import SamplingMetadata
43
-from vllm.sequence import SamplerOutput
43
+from vllm.sequence import IntermediateTensors, SamplerOutput
44
from vllm.transformers_utils.configs import JAISConfig
45
46
@@ -289,6 +289,7 @@ class JAISLMHeadModel(nn.Module):
289
positions: torch.Tensor,
290
kv_caches: List[torch.Tensor],
291
attn_metadata: AttentionMetadata,
292
+ intermediate_tensors: Optional[IntermediateTensors] = None,
293
) -> torch.Tensor:
294
hidden_states = self.transformer(input_ids, positions, kv_caches,
295
attn_metadata)
vllm/model_executor/models/llama.py
+73
-28
@@ -21,7 +21,7 @@
21
# See the License for the specific language governing permissions and
22
# limitations under the License.
23
"""Inference-only LLaMA model compatible with HuggingFace weights."""
24
-from typing import Any, Dict, Iterable, List, Optional, Tuple
24
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
25
26
import torch
27
from torch import nn
@@ -29,7 +29,8 @@ from transformers import LlamaConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.config import CacheConfig, LoRAConfig
32
-from vllm.distributed import (get_tensor_model_parallel_rank,
32
+from vllm.distributed import (get_pp_group, get_pp_indices,
33
+ get_tensor_model_parallel_rank,
34
get_tensor_model_parallel_world_size)
35
from vllm.model_executor.layers.activation import SiluAndMul
36
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -46,7 +47,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
47
from vllm.model_executor.model_loader.weight_utils import (
48
default_weight_loader, kv_cache_scales_loader)
49
from vllm.model_executor.sampling_metadata import SamplingMetadata
49
-from vllm.sequence import SamplerOutput
50
+from vllm.sequence import IntermediateTensors, SamplerOutput
51
from vllm.utils import is_hip, print_warning_once
52
53
from .interfaces import SupportsLoRA
@@ -261,12 +262,20 @@ class LlamaModel(nn.Module):
262
config.hidden_size,
263
org_num_embeddings=config.vocab_size,
264
)
264
- self.layers = nn.ModuleList([
265
- LlamaDecoderLayer(config=config,
266
- cache_config=cache_config,
267
- quant_config=quant_config)
268
- for idx in range(config.num_hidden_layers)
269
- ])
265
+ self.start_layer, self.end_layer = get_pp_indices(
266
+ config.num_hidden_layers,
267
+ get_pp_group().rank_in_group,
268
+ get_pp_group().world_size)
269
+ self.layers = nn.ModuleList(
270
+ [nn.Identity() for _ in range(self.start_layer)] + [
271
+ LlamaDecoderLayer(config=config,
272
+ cache_config=cache_config,
273
+ quant_config=quant_config)
274
+ for _ in range(self.start_layer, self.end_layer)
275
+ ] + [
276
+ nn.Identity()
277
+ for _ in range(self.end_layer, config.num_hidden_layers)
278
+ ])
279
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
280
281
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
@@ -278,22 +287,36 @@ class LlamaModel(nn.Module):
287
positions: torch.Tensor,
288
kv_caches: List[torch.Tensor],
289
attn_metadata: AttentionMetadata,
290
+ intermediate_tensors: Optional[IntermediateTensors],
291
inputs_embeds: Optional[torch.Tensor] = None,
282
- ) -> torch.Tensor:
283
- if inputs_embeds is not None:
284
- hidden_states = inputs_embeds
292
+ ) -> Union[torch.Tensor, IntermediateTensors]:
293
+ if get_pp_group().is_first_rank:
294
+ if inputs_embeds is not None:
295
+ hidden_states = inputs_embeds
296
+ else:
297
+ hidden_states = self.get_input_embeddings(input_ids)
298
+ residual = None
299
else:
286
- hidden_states = self.get_input_embeddings(input_ids)
287
- residual = None
288
- for i in range(len(self.layers)):
300
+ assert intermediate_tensors is not None
301
+ hidden_states = intermediate_tensors["hidden_states"]
302
+ residual = intermediate_tensors["residual"]
303
+
304
+ for i in range(self.start_layer, self.end_layer):
305
layer = self.layers[i]
306
hidden_states, residual = layer(
307
positions,
308
hidden_states,
293
- kv_caches[i],
309
+ kv_caches[i - self.start_layer],
310
attn_metadata,
311
residual,
312
)
313
+
314
+ if not get_pp_group().is_last_rank:
315
+ return IntermediateTensors({
316
+ "hidden_states": hidden_states,
317
+ "residual": residual
318
+ })
319
+
320
hidden_states, _ = self.norm(hidden_states, residual)
321
return hidden_states
322
@@ -372,10 +395,11 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
395
positions: torch.Tensor,
396
kv_caches: List[torch.Tensor],
397
attn_metadata: AttentionMetadata,
375
- ) -> torch.Tensor:
376
- hidden_states = self.model(input_ids, positions, kv_caches,
377
- attn_metadata)
378
- return hidden_states
398
+ intermediate_tensors: Optional[IntermediateTensors] = None,
399
+ ) -> Union[torch.Tensor, IntermediateTensors]:
400
+ model_output = self.model(input_ids, positions, kv_caches,
401
+ attn_metadata, intermediate_tensors)
402
+ return model_output
403
404
def compute_logits(self, hidden_states: torch.Tensor,
405
sampling_metadata: SamplingMetadata) -> torch.Tensor:
@@ -391,6 +415,20 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
415
next_tokens = self.sampler(logits, sampling_metadata)
416
return next_tokens
417
418
+ def make_empty_intermediate_tensors(
419
+ self, batch_size: int, dtype: torch.dtype,
420
+ device: torch.device) -> IntermediateTensors:
421
+ return IntermediateTensors({
422
+ "hidden_states":
423
+ torch.zeros((batch_size, self.config.hidden_size),
424
+ dtype=dtype,
425
+ device=device),
426
+ "residual":
427
+ torch.zeros((batch_size, self.config.hidden_size),
428
+ dtype=dtype,
429
+ device=device),
430
+ })
431
+
432
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
433
stacked_params_mapping = [
434
# (param_name, shard_name, shard_id)
@@ -416,9 +454,12 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
454
# Skip loading extra bias for GPTQ models.
455
if name.endswith(".bias") and name not in params_dict:
456
continue
419
- param = params_dict[name]
420
- weight_loader = param.weight_loader
421
- weight_loader(param, loaded_weight, shard_id)
457
+ try:
458
+ param = params_dict[name]
459
+ weight_loader = param.weight_loader
460
+ weight_loader(param, loaded_weight, shard_id)
461
+ except KeyError:
462
+ pass
463
break
464
else:
465
# Skip loading extra bias for GPTQ models.
@@ -437,10 +478,13 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
478
continue
479
else:
480
name = remapped_kv_scale_name
440
- param = params_dict[name]
441
- weight_loader = getattr(param, "weight_loader",
442
- default_weight_loader)
443
- weight_loader(param, loaded_weight)
481
+ try:
482
+ param = params_dict[name]
483
+ weight_loader = getattr(param, "weight_loader",
484
+ default_weight_loader)
485
+ weight_loader(param, loaded_weight)
486
+ except KeyError:
487
+ pass
488
489
# If this function is called, it should always initialize KV cache scale
490
# factors (or else raise an exception). Thus, handled exceptions should
@@ -452,7 +496,8 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
496
quantization_param_path, tp_rank, tp_size,
497
self.config.num_hidden_layers,
498
self.config.__class__.model_type):
455
- layer_self_attn = self.model.layers[layer_idx].self_attn
499
+ if not isinstance(self.model.layers[layer_idx], nn.Identity):
500
+ layer_self_attn = self.model.layers[layer_idx].self_attn
501
502
if is_hip():
503
# The scaling factor convention we are assuming is
vllm/model_executor/models/llava.py
+3
-1
@@ -18,7 +18,7 @@ from vllm.model_executor.models.clip import CLIPVisionModel
18
from vllm.model_executor.models.llama import LlamaModel
19
from vllm.model_executor.sampling_metadata import SamplingMetadata
20
from vllm.multimodal import MULTIMODAL_REGISTRY
21
-from vllm.sequence import SamplerOutput
21
+from vllm.sequence import IntermediateTensors, SamplerOutput
22
23
from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
24
from .interfaces import SupportsVision
@@ -202,6 +202,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
202
positions: torch.Tensor,
203
kv_caches: List[torch.Tensor],
204
attn_metadata: AttentionMetadata,
205
+ intermediate_tensors: Optional[IntermediateTensors] = None,
206
**kwargs: object,
207
) -> SamplerOutput:
208
"""Run forward pass for LLaVA-1.5.
@@ -247,6 +248,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
248
positions,
249
kv_caches,
250
attn_metadata,
251
+ None,
252
inputs_embeds=inputs_embeds)
253
254
return hidden_states
vllm/model_executor/models/llava_next.py
+3
-1
@@ -22,7 +22,7 @@ from vllm.model_executor.models.clip import CLIPVisionModel
22
from vllm.model_executor.models.llama import LlamaModel
23
from vllm.model_executor.sampling_metadata import SamplingMetadata
24
from vllm.multimodal import MULTIMODAL_REGISTRY
25
-from vllm.sequence import SamplerOutput
25
+from vllm.sequence import IntermediateTensors, SamplerOutput
26
27
from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
28
get_clip_patch_grid_length)
@@ -376,6 +376,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
376
positions: torch.Tensor,
377
kv_caches: List[torch.Tensor],
378
attn_metadata: AttentionMetadata,
379
+ intermediate_tensors: Optional[IntermediateTensors] = None,
380
**kwargs: object,
381
) -> SamplerOutput:
382
"""Run forward pass for LlaVA-NeXT.
@@ -430,6 +431,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
431
positions,
432
kv_caches,
433
attn_metadata,
434
+ None,
435
inputs_embeds=inputs_embeds)
436
437
return hidden_states
vllm/model_executor/models/minicpm.py
+2
-1
@@ -50,7 +50,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
50
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
51
from vllm.model_executor.sampling_metadata import SamplingMetadata
52
from vllm.model_executor.utils import set_weight_attrs
53
-from vllm.sequence import SamplerOutput
53
+from vllm.sequence import IntermediateTensors, SamplerOutput
54
55
from .interfaces import SupportsLoRA
56
@@ -462,6 +462,7 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
462
positions: torch.Tensor,
463
kv_caches: List[torch.Tensor],
464
attn_metadata: AttentionMetadata,
465
+ intermediate_tensors: Optional[IntermediateTensors] = None,
466
) -> torch.Tensor:
467
hidden_states = self.model(input_ids, positions, kv_caches,
468
attn_metadata)
vllm/model_executor/models/mixtral.py
+2
-1
@@ -51,7 +51,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
51
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
52
from vllm.model_executor.sampling_metadata import SamplingMetadata
53
from vllm.model_executor.utils import set_weight_attrs
54
-from vllm.sequence import SamplerOutput
54
+from vllm.sequence import IntermediateTensors, SamplerOutput
55
from vllm.utils import print_warning_once
56
57
from .interfaces import SupportsLoRA
@@ -536,6 +536,7 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
536
positions: torch.Tensor,
537
kv_caches: List[torch.Tensor],
538
attn_metadata: AttentionMetadata,
539
+ intermediate_tensors: Optional[IntermediateTensors] = None,
540
) -> torch.Tensor:
541
hidden_states = self.model(input_ids, positions, kv_caches,
542
attn_metadata)
vllm/model_executor/models/mixtral_quant.py
+2
-1
@@ -47,7 +47,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
47
ParallelLMHead, VocabParallelEmbedding)
48
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
49
from vllm.model_executor.sampling_metadata import SamplingMetadata
50
-from vllm.sequence import SamplerOutput
50
+from vllm.sequence import IntermediateTensors, SamplerOutput
51
52
53
class MixtralMLP(nn.Module):
@@ -354,6 +354,7 @@ class MixtralForCausalLM(nn.Module):
354
positions: torch.Tensor,
355
kv_caches: List[torch.Tensor],
356
attn_metadata: AttentionMetadata,
357
+ intermediate_tensors: Optional[IntermediateTensors] = None,
358
) -> torch.Tensor:
359
hidden_states = self.model(input_ids, positions, kv_caches,
360
attn_metadata)
vllm/model_executor/models/mpt.py
+2
-1
@@ -22,7 +22,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
22
VocabParallelEmbedding)
23
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24
from vllm.model_executor.sampling_metadata import SamplingMetadata
25
-from vllm.sequence import SamplerOutput
25
+from vllm.sequence import IntermediateTensors, SamplerOutput
26
from vllm.transformers_utils.configs.mpt import MPTConfig
27
28
@@ -273,6 +273,7 @@ class MPTForCausalLM(nn.Module):
273
positions: torch.Tensor,
274
kv_caches: List[torch.Tensor],
275
attn_metadata: AttentionMetadata,
276
+ intermediate_tensors: Optional[IntermediateTensors] = None,
277
) -> torch.Tensor:
278
hidden_states = self.transformer(input_ids, positions, kv_caches,
279
attn_metadata)
vllm/model_executor/models/olmo.py
+2
-1
@@ -43,7 +43,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
43
ParallelLMHead, VocabParallelEmbedding)
44
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45
from vllm.model_executor.sampling_metadata import SamplingMetadata
46
-from vllm.sequence import SamplerOutput
46
+from vllm.sequence import IntermediateTensors, SamplerOutput
47
48
49
class OlmoAttention(nn.Module):
@@ -301,6 +301,7 @@ class OlmoForCausalLM(nn.Module):
301
positions: torch.Tensor,
302
kv_caches: List[torch.Tensor],
303
attn_metadata: AttentionMetadata,
304
+ intermediate_tensors: Optional[IntermediateTensors] = None,
305
) -> torch.Tensor:
306
hidden_states = self.model(
307
input_ids=input_ids,
vllm/model_executor/models/opt.py
+2
-1
@@ -39,7 +39,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
39
VocabParallelEmbedding)
40
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41
from vllm.model_executor.sampling_metadata import SamplingMetadata
42
-from vllm.sequence import SamplerOutput
42
+from vllm.sequence import IntermediateTensors, SamplerOutput
43
44
45
class OPTLearnedPositionalEmbedding(nn.Embedding):
@@ -304,6 +304,7 @@ class OPTForCausalLM(nn.Module):
304
positions: torch.Tensor,
305
kv_caches: List[torch.Tensor],
306
attn_metadata: AttentionMetadata,
307
+ intermediate_tensors: Optional[IntermediateTensors] = None,
308
) -> torch.Tensor:
309
hidden_states = self.model(input_ids, positions, kv_caches,
310
attn_metadata)
vllm/model_executor/models/orion.py
+2
-1
@@ -26,7 +26,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
26
ParallelLMHead, VocabParallelEmbedding)
27
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
28
from vllm.model_executor.sampling_metadata import SamplingMetadata
29
-from vllm.sequence import SamplerOutput
29
+from vllm.sequence import IntermediateTensors, SamplerOutput
30
31
32
class OrionMLP(nn.Module):
@@ -269,6 +269,7 @@ class OrionForCausalLM(nn.Module):
269
positions: torch.Tensor,
270
kv_caches: List[torch.Tensor],
271
attn_metadata: AttentionMetadata,
272
+ intermediate_tensors: Optional[IntermediateTensors] = None,
273
) -> torch.Tensor:
274
hidden_states = self.model(input_ids, positions, kv_caches,
275
attn_metadata)
vllm/model_executor/models/phi.py
+2
-1
@@ -57,7 +57,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
57
ParallelLMHead, VocabParallelEmbedding)
58
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
59
from vllm.model_executor.sampling_metadata import SamplingMetadata
60
-from vllm.sequence import SamplerOutput
60
+from vllm.sequence import IntermediateTensors, SamplerOutput
61
62
from .interfaces import SupportsLoRA
63
@@ -278,6 +278,7 @@ class PhiForCausalLM(nn.Module, SupportsLoRA):
278
positions: torch.Tensor,
279
kv_caches: List[torch.Tensor],
280
attn_metadata: AttentionMetadata,
281
+ intermediate_tensors: Optional[IntermediateTensors] = None,
282
) -> torch.Tensor:
283
hidden_states = self.model(input_ids, positions, kv_caches,
284
attn_metadata)
vllm/model_executor/models/phi3_small.py
+2
-1
@@ -21,7 +21,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
21
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
22
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23
from vllm.model_executor.sampling_metadata import SamplingMetadata
24
-from vllm.sequence import SamplerOutput
24
+from vllm.sequence import IntermediateTensors, SamplerOutput
25
26
27
def load_column_parallel_weight(param: torch.nn.Parameter,
@@ -412,6 +412,7 @@ class Phi3SmallForCausalLM(nn.Module):
412
positions: Optional[torch.LongTensor],
413
kv_caches: List[torch.Tensor],
414
attn_metadata: AttentionMetadata,
415
+ intermediate_tensors: Optional[IntermediateTensors] = None,
416
) -> torch.Tensor:
417
output_hidden_states = self.model(
418
input_ids=input_ids,
vllm/model_executor/models/phi3v.py
+8
-3
@@ -35,7 +35,7 @@ from vllm.model_executor.models.clip import CLIPVisionModel
35
from vllm.model_executor.models.llama import LlamaModel
36
from vllm.model_executor.sampling_metadata import SamplingMetadata
37
from vllm.multimodal import MULTIMODAL_REGISTRY
38
-from vllm.sequence import SamplerOutput
38
+from vllm.sequence import IntermediateTensors, SamplerOutput
39
40
from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
41
from .interfaces import SupportsVision
@@ -381,9 +381,13 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
381
382
return None
383
384
- def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
384
+ def forward(self,
385
+ input_ids: torch.Tensor,
386
+ positions: torch.Tensor,
387
kv_caches: List[torch.Tensor],
386
- attn_metadata: AttentionMetadata, **kwargs: object):
388
+ attn_metadata: AttentionMetadata,
389
+ intermediate_tensors: Optional[IntermediateTensors] = None,
390
+ **kwargs: object):
391
image_input = self._parse_and_validate_image_input(**kwargs)
392
393
if image_input is not None:
@@ -398,6 +402,7 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
402
positions,
403
kv_caches,
404
attn_metadata,
405
+ intermediate_tensors,
406
inputs_embeds=inputs_embeds)
407
408
return hidden_states
vllm/model_executor/models/qwen.py
+2
-1
@@ -27,7 +27,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
27
ParallelLMHead, VocabParallelEmbedding)
28
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
29
from vllm.model_executor.sampling_metadata import SamplingMetadata
30
-from vllm.sequence import SamplerOutput
30
+from vllm.sequence import IntermediateTensors, SamplerOutput
31
from vllm.utils import print_warning_once
32
33
@@ -245,6 +245,7 @@ class QWenLMHeadModel(nn.Module):
245
positions: torch.Tensor,
246
kv_caches: List[torch.Tensor],
247
attn_metadata: AttentionMetadata,
248
+ intermediate_tensors: Optional[IntermediateTensors] = None,
249
) -> torch.Tensor:
250
hidden_states = self.transformer(input_ids, positions, kv_caches,
251
attn_metadata)
vllm/model_executor/models/qwen2.py
+2
-1
@@ -45,7 +45,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
45
ParallelLMHead, VocabParallelEmbedding)
46
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
47
from vllm.model_executor.sampling_metadata import SamplingMetadata
48
-from vllm.sequence import SamplerOutput
48
+from vllm.sequence import IntermediateTensors, SamplerOutput
49
from vllm.utils import print_warning_once
50
51
from .interfaces import SupportsLoRA
@@ -331,6 +331,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
331
positions: torch.Tensor,
332
kv_caches: List[torch.Tensor],
333
attn_metadata: AttentionMetadata,
334
+ intermediate_tensors: Optional[IntermediateTensors] = None,
335
) -> torch.Tensor:
336
hidden_states = self.model(input_ids, positions, kv_caches,
337
attn_metadata)
vllm/model_executor/models/qwen2_moe.py
+2
-1
@@ -50,7 +50,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
50
ParallelLMHead, VocabParallelEmbedding)
51
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
52
from vllm.model_executor.sampling_metadata import SamplingMetadata
53
-from vllm.sequence import SamplerOutput
53
+from vllm.sequence import IntermediateTensors, SamplerOutput
54
55
56
class Qwen2MoeMLP(nn.Module):
@@ -397,6 +397,7 @@ class Qwen2MoeForCausalLM(nn.Module):
397
positions: torch.Tensor,
398
kv_caches: List[torch.Tensor],
399
attn_metadata: AttentionMetadata,
400
+ intermediate_tensors: Optional[IntermediateTensors] = None,
401
) -> torch.Tensor:
402
hidden_states = self.model(input_ids, positions, kv_caches,
403
attn_metadata)
vllm/model_executor/models/stablelm.py
+2
-1
@@ -41,7 +41,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
41
ParallelLMHead, VocabParallelEmbedding)
42
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
43
from vllm.model_executor.sampling_metadata import SamplingMetadata
44
-from vllm.sequence import SamplerOutput
44
+from vllm.sequence import IntermediateTensors, SamplerOutput
45
46
47
class StablelmMLP(nn.Module):
@@ -250,6 +250,7 @@ class StablelmForCausalLM(nn.Module):
250
positions: torch.Tensor,
251
kv_caches: List[torch.Tensor],
252
attn_metadata: AttentionMetadata,
253
+ intermediate_tensors: Optional[IntermediateTensors] = None,
254
) -> torch.Tensor:
255
hidden_states = self.model(input_ids, positions, kv_caches,
256
attn_metadata)
vllm/model_executor/models/starcoder2.py
+2
-1
@@ -40,7 +40,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
40
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
41
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
42
from vllm.model_executor.sampling_metadata import SamplingMetadata
43
-from vllm.sequence import SamplerOutput
43
+from vllm.sequence import IntermediateTensors, SamplerOutput
44
45
46
class Starcoder2Attention(nn.Module):
@@ -262,6 +262,7 @@ class Starcoder2ForCausalLM(nn.Module):
262
positions: torch.Tensor,
263
kv_caches: List[torch.Tensor],
264
attn_metadata: AttentionMetadata,
265
+ intermediate_tensors: Optional[IntermediateTensors] = None,
266
) -> torch.Tensor:
267
hidden_states = self.model(input_ids, positions, kv_caches,
268
attn_metadata)
vllm/model_executor/models/xverse.py
+2
-1
@@ -43,7 +43,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
43
ParallelLMHead, VocabParallelEmbedding)
44
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45
from vllm.model_executor.sampling_metadata import SamplingMetadata
46
-from vllm.sequence import SamplerOutput
46
+from vllm.sequence import IntermediateTensors, SamplerOutput
47
48
from .interfaces import SupportsLoRA
49
@@ -320,6 +320,7 @@ class XverseForCausalLM(nn.Module, SupportsLoRA):
320
positions: torch.Tensor,
321
kv_caches: List[torch.Tensor],
322
attn_metadata: AttentionMetadata,
323
+ intermediate_tensors: Optional[IntermediateTensors] = None,
324
) -> torch.Tensor:
325
hidden_states = self.model(input_ids, positions, kv_caches,
326
attn_metadata)
vllm/sequence.py
+31
@@ -770,6 +770,34 @@ class EmbeddingSequenceGroupOutput(SequenceGroupOutput):
770
return self.embeddings == other.embeddings
771
772
773
+@dataclass
774
+class IntermediateTensors:
775
+ """For all pipeline stages except the last, we need to return the hidden
776
+ states and residuals to be sent to the next stage. This data structure
777
+ contains the hidden states and residuals for a request.
778
+ """
779
+
780
+ tensors: Dict[str, torch.Tensor]
781
+
782
+ def __getitem__(self, key: Union[str, slice]):
783
+ if isinstance(key, str):
784
+ return self.tensors[key]
785
+ elif isinstance(key, slice):
786
+ return self.__class__({k: v[key] for k, v in self.tensors.items()})
787
+
788
+ def __setitem__(self, key: str, value):
789
+ self.tensors[key] = value
790
+
791
+ def __len__(self):
792
+ return len(self.tensors)
793
+
794
+ def __eq__(self, other: object):
795
+ return isinstance(other, self.__class__) and self
796
+
797
+ def __repr__(self) -> str:
798
+ return f"IntermediateTensors(tensors={self.tensors})"
799
+
800
+
801
@dataclass
802
class SamplerOutput:
803
"""For each sequence group, we generate a list of SequenceOutput object,
@@ -896,6 +924,8 @@ class ExecuteModelRequest:
924
blocks_to_swap_out: List[Tuple[int, int]] = field(default_factory=list)
925
# Blocks to copy. Source to dest block.
926
blocks_to_copy: List[Tuple[int, int]] = field(default_factory=list)
927
+ # Virtual engine ID for pipeline parallel.
928
+ virtual_engine: int = 0
929
# The number of slots for lookahead decoding.
930
num_lookahead_slots: int = 0
931
# The number of requests in the running queue.
@@ -914,6 +944,7 @@ class ExecuteModelRequest:
944
blocks_to_swap_in=self.blocks_to_swap_in.copy(),
945
blocks_to_swap_out=self.blocks_to_swap_out.copy(),
946
blocks_to_copy=self.blocks_to_copy.copy(),
947
+ virtual_engine=self.virtual_engine,
948
num_lookahead_slots=self.num_lookahead_slots,
949
running_queue_size=self.running_queue_size,
950
previous_hidden_states=self.previous_hidden_states,
vllm/spec_decode/draft_model_runner.py
+10
-5
@@ -6,7 +6,8 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6
ModelConfig, ParallelConfig, SchedulerConfig,
7
VisionLanguageConfig)
8
from vllm.logger import init_logger
9
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
9
+from vllm.sequence import (IntermediateTensors, SamplerOutput,
10
+ SequenceGroupMetadata)
11
from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
12
ModelRunner)
13
@@ -74,9 +75,9 @@ class TP1DraftModelRunner(ModelRunner):
75
List[SequenceGroupMetadata]] = None
76
77
def prepare_model_input(
77
- self,
78
- seq_group_metadata_list: List[SequenceGroupMetadata],
79
- ) -> ModelInputForGPUWithSamplingMetadata:
78
+ self,
79
+ seq_group_metadata_list: List[SequenceGroupMetadata],
80
+ virtual_engine: int = 0) -> ModelInputForGPUWithSamplingMetadata:
81
"""A temporary solution that caches the seq_group_metadata_list
82
for multi-step execution.
83
TODO: In-place update model_input and remove this function.
@@ -115,6 +116,7 @@ class TP1DraftModelRunner(ModelRunner):
116
self,
117
model_input: ModelInputForGPUWithSamplingMetadata,
118
kv_caches: List[torch.Tensor],
119
+ intermediate_tensors: Optional[IntermediateTensors] = None,
120
num_steps: int = 1,
121
) -> Optional[List[SamplerOutput]]:
122
# Since we do not broadcast data inside execute_model anymore,
@@ -130,6 +132,7 @@ class TP1DraftModelRunner(ModelRunner):
132
self.set_active_loras(model_input.lora_requests,
133
model_input.lora_mapping)
134
135
+ virtual_engine = model_input.virtual_engine
136
outputs: List[SamplerOutput] = []
137
for step in range(num_steps):
138
# Currently cuda graph is only supported by the decode phase.
@@ -139,7 +142,8 @@ class TP1DraftModelRunner(ModelRunner):
142
if prefill_meta is None and decode_meta.use_cuda_graph:
143
assert model_input.input_tokens is not None
144
graph_batch_size = model_input.input_tokens.shape[0]
142
- model_executable = self.graph_runners[graph_batch_size]
145
+ model_executable = (
146
+ self.graph_runners[virtual_engine][graph_batch_size])
147
else:
148
model_executable = self.model
149
@@ -149,6 +153,7 @@ class TP1DraftModelRunner(ModelRunner):
153
positions=model_input.input_positions,
154
kv_caches=kv_caches,
155
attn_metadata=model_input.attn_metadata,
156
+ intermediate_tensors=intermediate_tensors,
157
**multi_modal_kwargs,
158
)
159
vllm/worker/cache_engine.py
+4
@@ -38,7 +38,11 @@ class CacheEngine:
38
39
self.block_size = cache_config.block_size
40
self.num_gpu_blocks = cache_config.num_gpu_blocks
41
+ if self.num_gpu_blocks:
42
+ self.num_gpu_blocks //= parallel_config.pipeline_parallel_size
43
self.num_cpu_blocks = cache_config.num_cpu_blocks
44
+ if self.num_cpu_blocks:
45
+ self.num_cpu_blocks //= parallel_config.pipeline_parallel_size
46
47
if cache_config.cache_dtype == "auto":
48
self.dtype = model_config.dtype
vllm/worker/cpu_model_runner.py
+4
-1
@@ -13,7 +13,8 @@ from vllm.logger import init_logger
13
from vllm.model_executor import SamplingMetadata
14
from vllm.model_executor.model_loader import get_model
15
from vllm.multimodal import MULTIMODAL_REGISTRY
16
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
16
+from vllm.sequence import (IntermediateTensors, SamplerOutput,
17
+ SequenceGroupMetadata)
18
from vllm.utils import make_tensor_with_pad
19
from vllm.worker.model_runner_base import (
20
ModelRunnerBase, ModelRunnerInputBase,
@@ -315,6 +316,7 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
316
def prepare_model_input(
317
self,
318
seq_group_metadata_list: List[SequenceGroupMetadata],
319
+ virtual_engine: int = 0,
320
) -> CPUModelInput:
321
multi_modal_kwargs = None
322
# NOTE: We assume that all sequences in the group are all prompts or
@@ -351,6 +353,7 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
353
self,
354
model_input: CPUModelInput,
355
kv_caches: List[torch.Tensor],
356
+ intermediate_tensors: Optional[IntermediateTensors] = None,
357
num_steps: int = 1,
358
) -> Optional[List[SamplerOutput]]:
359
if num_steps > 1:
vllm/worker/cpu_worker.py
+24
-14
@@ -167,8 +167,8 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
167
is_driver_worker=is_driver_worker)
168
# Uninitialized cache engine. Will be initialized by
169
# initialize_cache.
170
- self.cache_engine: CPUCacheEngine
171
- self.cpu_cache: List[torch.Tensor]
170
+ self.cache_engine: List[CPUCacheEngine]
171
+ self.cpu_cache: List[List[torch.Tensor]]
172
173
def init_device(self) -> None:
174
self.init_distributed_environment()
@@ -242,25 +242,32 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
242
"initializing the engine.")
243
244
def _init_cache_engine(self) -> None:
245
- self.cache_engine = CPUCacheEngine(self.cache_config,
246
- self.model_config,
247
- self.parallel_config,
248
- self.device_config)
249
- self.cpu_cache = self.cache_engine.cpu_cache
250
- self.model_runner.block_size = self.cache_engine.block_size
251
-
252
- assert self.cpu_cache is not None
245
+ self.cache_engine = [
246
+ CPUCacheEngine(self.cache_config, self.model_config,
247
+ self.parallel_config, self.device_config)
248
+ for _ in range(self.parallel_config.pipeline_parallel_size)
249
+ ]
250
+ self.cpu_cache = [
251
+ self.cache_engine[ve].cpu_cache
252
+ for ve in range(self.parallel_config.pipeline_parallel_size)
253
+ ]
254
+ self.model_runner.block_size = self.cache_engine[0].block_size
255
+
256
+ assert all(
257
+ self.cpu_cache[ve] is not None
258
+ for ve in range(self.parallel_config.pipeline_parallel_size))
259
260
# Populate the cache to warmup the memory
255
- for layer_cache in self.cpu_cache:
256
- layer_cache.fill_(0)
261
+ for ve in range(self.parallel_config.pipeline_parallel_size):
262
+ for layer_cache in self.cpu_cache[ve]:
263
+ layer_cache.fill_(0)
264
265
@property
266
def do_metadata_broadcast(self) -> bool:
267
return self.parallel_config.tensor_parallel_size > 1
268
269
@property
263
- def kv_cache(self) -> Optional[List[torch.Tensor]]:
270
+ def kv_cache(self) -> Optional[List[List[torch.Tensor]]]:
271
return self.cpu_cache
272
273
def execute_worker(
@@ -269,12 +276,14 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
276
) -> None:
277
if (worker_input.blocks_to_copy is not None
278
and worker_input.blocks_to_copy.numel() > 0):
272
- self.cache_engine.copy(worker_input.blocks_to_copy)
279
+ self.cache_engine[worker_input.virtual_engine].copy(
280
+ worker_input.blocks_to_copy)
281
282
@torch.inference_mode()
283
def prepare_worker_input(
284
self, execute_model_req: ExecuteModelRequest) -> WorkerInput:
285
assert execute_model_req is not None
286
+ virtual_engine = execute_model_req.virtual_engine
287
num_seq_groups: int = len(execute_model_req.seq_group_metadata_list)
288
blocks_to_copy = execute_model_req.blocks_to_copy
289
blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
@@ -285,6 +294,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
294
return WorkerInput(
295
num_seq_groups=num_seq_groups,
296
blocks_to_copy=blocks_to_copy,
297
+ virtual_engine=virtual_engine,
298
)
299
300
def init_distributed_environment(self) -> None:
vllm/worker/embedding_model_runner.py
+7
-2
@@ -9,7 +9,8 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
9
from vllm.logger import init_logger
10
from vllm.model_executor.pooling_metadata import PoolingMetadata
11
from vllm.pooling_params import PoolingParams
12
-from vllm.sequence import PoolerOutput, SequenceData, SequenceGroupMetadata
12
+from vllm.sequence import (IntermediateTensors, PoolerOutput, SequenceData,
13
+ SequenceGroupMetadata)
14
from vllm.worker.model_runner import GPUModelRunnerBase, ModelInputForGPU
15
16
logger = init_logger(__name__)
@@ -57,6 +58,7 @@ class EmbeddingModelRunner(
58
self,
59
model_input: ModelInputForGPUWithPoolingMetadata,
60
kv_caches: List[torch.Tensor],
61
+ intermediate_tensors: Optional[IntermediateTensors] = None,
62
num_steps: int = 1,
63
) -> Optional[List[PoolerOutput]]:
64
if num_steps > 1:
@@ -73,10 +75,12 @@ class EmbeddingModelRunner(
75
assert model_input.attn_metadata is not None
76
prefill_meta = model_input.attn_metadata.prefill_metadata
77
decode_meta = model_input.attn_metadata.decode_metadata
78
+ virtual_engine = model_input.virtual_engine
79
if prefill_meta is None and decode_meta.use_cuda_graph:
80
assert model_input.input_tokens is not None
81
graph_batch_size = model_input.input_tokens.shape[0]
79
- model_executable = self.graph_runners[graph_batch_size]
82
+ model_executable = self.graph_runners[virtual_engine][
83
+ graph_batch_size]
84
else:
85
model_executable = self.model
86
@@ -115,6 +119,7 @@ class EmbeddingModelRunner(
119
def prepare_model_input(
120
self,
121
seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
122
+ virtual_engine: int = 0,
123
) -> ModelInputForGPUWithPoolingMetadata:
124
assert seq_group_metadata_list is not None
125
model_input = self._prepare_model_input_tensors(
vllm/worker/model_runner.py
+201
-125
@@ -8,6 +8,7 @@ from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Type,
8
9
import numpy as np
10
import torch
11
+import torch.distributed
12
import torch.nn as nn
13
14
try:
@@ -25,6 +26,7 @@ from vllm.attention import AttentionMetadata, get_attn_backend
26
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
27
ModelConfig, ParallelConfig, SchedulerConfig,
28
VisionLanguageConfig)
29
+from vllm.distributed import get_pp_group
30
from vllm.distributed.parallel_state import graph_capture
31
from vllm.inputs import INPUT_REGISTRY
32
from vllm.logger import init_logger
@@ -37,7 +39,8 @@ from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
39
from vllm.model_executor.models.interfaces import supports_lora
40
from vllm.multimodal import MULTIMODAL_REGISTRY
41
from vllm.sampling_params import SamplingParams
40
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
42
+from vllm.sequence import (IntermediateTensors, SamplerOutput,
43
+ SequenceGroupMetadata)
44
from vllm.utils import (CudaMemoryProfiler, get_kv_cache_torch_dtype, is_hip,
45
is_pin_memory_available, make_tensor_with_pad)
46
from vllm.worker.model_runner_base import (
@@ -81,6 +84,7 @@ class ModelInputForGPU(ModelRunnerInputBase):
84
lora_requests: Optional[Set[LoRARequest]] = None
85
attn_metadata: Optional["AttentionMetadata"] = None
86
multi_modal_kwargs: Optional[Dict[str, torch.Tensor]] = None
87
+ virtual_engine: int = 0
88
89
def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
90
tensor_dict = {
@@ -89,6 +93,7 @@ class ModelInputForGPU(ModelRunnerInputBase):
93
"lora_requests": self.lora_requests,
94
"lora_mapping": self.lora_mapping,
95
"multi_modal_kwargs": self.multi_modal_kwargs,
96
+ "virtual_engine": self.virtual_engine,
97
}
98
_add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
99
return tensor_dict
@@ -122,6 +127,7 @@ class ModelInputForGPUWithSamplingMetadata(ModelInputForGPU):
127
"lora_requests": self.lora_requests,
128
"lora_mapping": self.lora_mapping,
129
"multi_modal_kwargs": self.multi_modal_kwargs,
130
+ "virtual_engine": self.virtual_engine,
131
}
132
_add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
133
_add_sampling_metadata_broadcastable_dict(tensor_dict,
@@ -179,7 +185,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
185
self.sliding_window = model_config.get_sliding_window()
186
self.block_size = cache_config.block_size
187
self.max_seq_len_to_capture = self.model_config.max_seq_len_to_capture
182
- self.graph_runners: Dict[int, CUDAGraphRunner] = {}
188
+
189
+ self.graph_runners: List[Dict[int, CUDAGraphRunner]] = [
190
+ {} for _ in range(self.parallel_config.pipeline_parallel_size)
191
+ ]
192
self.graph_memory_pool: Optional[Tuple[
193
int, int]] = None # Set during graph capture.
194
# When using CUDA graph, the input block tables must be padded to
@@ -787,9 +796,11 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
796
max_num_seqs = min(
797
max_num_seqs,
798
int(max_num_batched_tokens / vlm_config.image_feature_size))
799
+ batch_size = 0
800
for group_id in range(max_num_seqs):
801
seq_len = (max_num_batched_tokens // max_num_seqs +
802
(group_id < max_num_batched_tokens % max_num_seqs))
803
+ batch_size += seq_len
804
805
seq_data, dummy_multi_modal_data = INPUT_REGISTRY \
806
.dummy_data_for_profiling(model_config, seq_len)
@@ -811,7 +822,13 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
822
num_layers = self.model_config.get_num_layers(self.parallel_config)
823
kv_caches = [None] * num_layers
824
model_input = self.prepare_model_input(seqs)
814
- self.execute_model(model_input, kv_caches)
825
+ intermediate_tensors = None
826
+ if not get_pp_group().is_first_rank:
827
+ intermediate_tensors = self.model.make_empty_intermediate_tensors(
828
+ batch_size=batch_size,
829
+ dtype=self.model_config.dtype,
830
+ device=self.device)
831
+ self.execute_model(model_input, kv_caches, intermediate_tensors)
832
torch.cuda.synchronize()
833
return
834
@@ -847,7 +864,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
864
return self.lora_manager.list_loras()
865
866
@torch.inference_mode()
850
- def capture_model(self, kv_caches: List[torch.Tensor]) -> None:
867
+ def capture_model(self, kv_caches: List[List[torch.Tensor]]) -> None:
868
"""Cuda graph capture a model.
869
870
Note that CUDA graph's performance gain is negligible if number
@@ -880,10 +897,18 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
897
slot_mapping.fill_(_PAD_SLOT_ID)
898
seq_lens = torch.ones(max_batch_size, dtype=torch.int32).cuda()
899
block_tables = torch.from_numpy(self.graph_block_tables).cuda()
900
+ intermediate_inputs = None
901
+ if not get_pp_group().is_first_rank:
902
+ intermediate_inputs = self.model.make_empty_intermediate_tensors(
903
+ batch_size=max_batch_size,
904
+ dtype=self.model_config.dtype,
905
+ device=self.device)
906
907
# Prepare buffer for outputs. These will be reused for all batch sizes.
908
# It will be filled after the first graph capture.
886
- hidden_states: Optional[torch.Tensor] = None
909
+ hidden_or_intermediate_states: List[Optional[torch.Tensor]] = [
910
+ None
911
+ ] * self.parallel_config.pipeline_parallel_size
912
913
graph_batch_size = _get_graph_batch_size(
914
self.scheduler_config.max_num_seqs)
@@ -912,109 +937,120 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
937
with graph_capture() as graph_capture_context:
938
# NOTE: Capturing the largest batch size first may help reduce the
939
# memory usage of CUDA graph.
915
- for batch_size in reversed(batch_size_capture_list):
916
- if self.attn_backend.get_name() == "flashinfer":
917
- indptr_buffer = indptr_buffer[:batch_size + 1]
918
- last_page_len_buffer = last_page_len_buffer[:batch_size]
919
-
920
- num_qo_heads = self.model_config.get_num_attention_heads(
921
- self.parallel_config)
922
- num_kv_heads = self.model_config.get_num_kv_heads(
923
- self.parallel_config)
924
- if num_qo_heads // num_kv_heads >= 4:
925
- use_tensor_cores = True
940
+ for virtual_engine in range(
941
+ self.parallel_config.pipeline_parallel_size):
942
+ for batch_size in reversed(batch_size_capture_list):
943
+ if self.attn_backend.get_name() == "flashinfer":
944
+ indptr_buffer = indptr_buffer[:batch_size + 1]
945
+ last_page_len_buffer = last_page_len_buffer[:
946
+ batch_size]
947
+
948
+ num_qo_heads = (
949
+ self.model_config.get_num_attention_heads(
950
+ self.parallel_config))
951
+ num_kv_heads = self.model_config.get_num_kv_heads(
952
+ self.parallel_config)
953
+ if num_qo_heads // num_kv_heads >= 4:
954
+ use_tensor_cores = True
955
+ else:
956
+ use_tensor_cores = False
957
+ decode_wrapper = \
958
+ CUDAGraphBatchDecodeWithPagedKVCacheWrapper(
959
+ decode_workspace_buffer, indptr_buffer,
960
+ indices_buffer, last_page_len_buffer, "NHD",
961
+ use_tensor_cores)
962
+ kv_cache_dtype = get_kv_cache_torch_dtype(
963
+ self.kv_cache_dtype, self.model_config.dtype)
964
+
965
+ paged_kv_indptr_tensor_host = torch.arange(
966
+ 0, batch_size + 1, dtype=torch.int32)
967
+ paged_kv_indices_tensor_host = torch.arange(
968
+ 0, batch_size, dtype=torch.int32)
969
+ paged_kv_last_page_len_tensor_host = torch.full(
970
+ (batch_size, ), self.block_size, dtype=torch.int32)
971
+ query_start_loc_host = torch.arange(0,
972
+ batch_size + 1,
973
+ dtype=torch.int32)
974
+
975
+ attn_metadata = self.attn_backend.make_metadata(
976
+ num_prefills=0,
977
+ slot_mapping=slot_mapping[:batch_size],
978
+ num_prefill_tokens=0,
979
+ num_decode_tokens=batch_size,
980
+ max_prefill_seq_len=0,
981
+ block_tables=block_tables,
982
+ paged_kv_indptr=paged_kv_indptr_tensor_host,
983
+ paged_kv_indices=paged_kv_indices_tensor_host,
984
+ paged_kv_last_page_len=
985
+ paged_kv_last_page_len_tensor_host,
986
+ num_qo_heads=num_qo_heads,
987
+ num_kv_heads=num_kv_heads,
988
+ head_dim=self.model_config.get_head_size(),
989
+ page_size=self.block_size,
990
+ seq_start_loc=None,
991
+ query_start_loc=query_start_loc_host,
992
+ device=self.device,
993
+ data_type=kv_cache_dtype,
994
+ use_cuda_graph=True,
995
+ decode_wrapper=decode_wrapper,
996
+ prefill_wrapper=None)
997
+ attn_metadata.begin_forward()
998
else:
927
- use_tensor_cores = False
928
- decode_wrapper = \
929
- CUDAGraphBatchDecodeWithPagedKVCacheWrapper(
930
- decode_workspace_buffer, indptr_buffer, indices_buffer,
931
- last_page_len_buffer, "NHD", use_tensor_cores)
932
- kv_cache_dtype = get_kv_cache_torch_dtype(
933
- self.kv_cache_dtype, self.model_config.dtype)
934
-
935
- paged_kv_indptr_tensor_host = torch.arange(
936
- 0, batch_size + 1, dtype=torch.int32)
937
- paged_kv_indices_tensor_host = torch.arange(
938
- 0, batch_size, dtype=torch.int32)
939
- paged_kv_last_page_len_tensor_host = torch.full(
940
- (batch_size, ), self.block_size, dtype=torch.int32)
941
- query_start_loc_host = torch.arange(0,
942
- batch_size + 1,
943
- dtype=torch.int32)
944
-
945
- attn_metadata = self.attn_backend.make_metadata(
946
- num_prefills=0,
947
- slot_mapping=slot_mapping[:batch_size],
948
- num_prefill_tokens=0,
949
- num_decode_tokens=batch_size,
950
- max_prefill_seq_len=0,
951
- block_tables=block_tables,
952
- paged_kv_indptr=paged_kv_indptr_tensor_host,
953
- paged_kv_indices=paged_kv_indices_tensor_host,
954
- paged_kv_last_page_len=
955
- paged_kv_last_page_len_tensor_host,
956
- num_qo_heads=num_qo_heads,
957
- num_kv_heads=num_kv_heads,
958
- head_dim=self.model_config.get_head_size(),
959
- page_size=self.block_size,
960
- seq_start_loc=None,
961
- query_start_loc=query_start_loc_host,
962
- device=self.device,
963
- data_type=kv_cache_dtype,
964
- use_cuda_graph=True,
965
- decode_wrapper=decode_wrapper,
966
- prefill_wrapper=None)
967
- attn_metadata.begin_forward()
968
- else:
969
- attn_metadata = self.attn_backend.make_metadata(
970
- num_prefills=0,
971
- num_prefill_tokens=0,
972
- num_decode_tokens=batch_size,
973
- slot_mapping=slot_mapping[:batch_size],
974
- seq_lens=None,
975
- seq_lens_tensor=seq_lens[:batch_size],
976
- max_query_len=None,
977
- max_prefill_seq_len=0,
978
- max_decode_seq_len=self.max_seq_len_to_capture,
979
- query_start_loc=None,
980
- seq_start_loc=None,
981
- context_lens_tensor=None,
982
- block_tables=block_tables[:batch_size],
983
- use_cuda_graph=True,
999
+ attn_metadata = self.attn_backend.make_metadata(
1000
+ num_prefills=0,
1001
+ num_prefill_tokens=0,
1002
+ num_decode_tokens=batch_size,
1003
+ slot_mapping=slot_mapping[:batch_size],
1004
+ seq_lens=None,
1005
+ seq_lens_tensor=seq_lens[:batch_size],
1006
+ max_query_len=None,
1007
+ max_prefill_seq_len=0,
1008
+ max_decode_seq_len=self.max_seq_len_to_capture,
1009
+ query_start_loc=None,
1010
+ seq_start_loc=None,
1011
+ context_lens_tensor=None,
1012
+ block_tables=block_tables[:batch_size],
1013
+ use_cuda_graph=True,
1014
+ )
1015
+
1016
+ if self.lora_config:
1017
+ lora_mapping = LoRAMapping(
1018
+ [0] * batch_size,
1019
+ [0] * batch_size,
1020
+ )
1021
+ self.set_active_loras(set(), lora_mapping)
1022
+
1023
+ graph_runner = CUDAGraphRunner(
1024
+ self.model, self.attn_backend.get_name())
1025
+
1026
+ if self.attn_backend.get_name() == "flashinfer":
1027
+ graph_runner.flashinfer_indptr_buffer = indptr_buffer
1028
+ graph_runner.flashinfer_indices_buffer = indices_buffer
1029
+ graph_runner.flashinfer_last_page_len_buffer = \
1030
+ last_page_len_buffer
1031
+ graph_runner.flashinfer_decode_workspace_buffer = \
1032
+ decode_workspace_buffer
1033
+ graph_runner.flashinfer_decode_wrapper = \
1034
+ decode_wrapper
1035
+
1036
+ graph_runner.capture(
1037
+ input_tokens[:batch_size],
1038
+ input_positions[:batch_size],
1039
+ hidden_or_intermediate_states[
1040
+ virtual_engine] # type: ignore
1041
+ [:batch_size]
1042
+ if hidden_or_intermediate_states[virtual_engine]
1043
+ is not None else None,
1044
+ intermediate_inputs[:batch_size]
1045
+ if intermediate_inputs is not None else None,
1046
+ kv_caches[virtual_engine],
1047
+ attn_metadata,
1048
+ memory_pool=self.graph_memory_pool,
1049
+ stream=graph_capture_context.stream,
1050
)
985
-
986
- if self.lora_config:
987
- lora_mapping = LoRAMapping(
988
- [0] * batch_size,
989
- [0] * batch_size,
990
- )
991
- self.set_active_loras(set(), lora_mapping)
992
-
993
- graph_runner = CUDAGraphRunner(self.model,
994
- self.attn_backend.get_name())
995
-
996
- if self.attn_backend.get_name() == "flashinfer":
997
- graph_runner.flashinfer_indptr_buffer = indptr_buffer
998
- graph_runner.flashinfer_indices_buffer = indices_buffer
999
- graph_runner.flashinfer_last_page_len_buffer = \
1000
- last_page_len_buffer
1001
- graph_runner.flashinfer_decode_workspace_buffer = \
1002
- decode_workspace_buffer
1003
- graph_runner.flashinfer_decode_wrapper = \
1004
- decode_wrapper
1005
-
1006
- graph_runner.capture(
1007
- input_tokens[:batch_size],
1008
- input_positions[:batch_size],
1009
- hidden_states[:batch_size]
1010
- if hidden_states is not None else None,
1011
- kv_caches,
1012
- attn_metadata,
1013
- memory_pool=self.graph_memory_pool,
1014
- stream=graph_capture_context.stream,
1015
- )
1016
- self.graph_memory_pool = graph_runner.graph.pool()
1017
- self.graph_runners[batch_size] = graph_runner
1051
+ self.graph_memory_pool = graph_runner.graph.pool()
1052
+ self.graph_runners[virtual_engine][batch_size] = (
1053
+ graph_runner)
1054
1055
end_time = time.perf_counter()
1056
elapsed_time = end_time - start_time
@@ -1047,6 +1083,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1083
def prepare_model_input(
1084
self,
1085
seq_group_metadata_list: List[SequenceGroupMetadata],
1086
+ virtual_engine: int = 0,
1087
) -> ModelInputForGPUWithSamplingMetadata:
1088
"""Prepare the model input based on a given sequence group, including
1089
metadata for the sampling step.
@@ -1072,15 +1109,17 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1109
if seq_group_metadata_list else None)
1110
return dataclasses.replace(model_input,
1111
sampling_metadata=sampling_metadata,
1075
- is_prompt=is_prompt)
1112
+ is_prompt=is_prompt,
1113
+ virtual_engine=virtual_engine)
1114
1115
@torch.inference_mode()
1116
def execute_model(
1117
self,
1118
model_input: ModelInputForGPUWithSamplingMetadata,
1119
kv_caches: List[torch.Tensor],
1120
+ intermediate_tensors: Optional[IntermediateTensors] = None,
1121
num_steps: int = 1,
1083
- ) -> Optional[List[SamplerOutput]]:
1122
+ ) -> Optional[Union[List[SamplerOutput], IntermediateTensors]]:
1123
if num_steps > 1:
1124
raise ValueError("num_steps > 1 is not supported in ModelRunner")
1125
@@ -1124,27 +1163,34 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1163
assert model_input.attn_metadata is not None
1164
prefill_meta = model_input.attn_metadata.prefill_metadata
1165
decode_meta = model_input.attn_metadata.decode_metadata
1166
+ # TODO(andoorve): We can remove this once all
1167
+ # virtual engines share the same kv cache.
1168
+ virtual_engine = model_input.virtual_engine
1169
if prefill_meta is None and decode_meta.use_cuda_graph:
1170
assert model_input.input_tokens is not None
1171
graph_batch_size = model_input.input_tokens.shape[0]
1130
- model_executable = self.graph_runners[graph_batch_size]
1172
+ model_executable = self.graph_runners[virtual_engine][
1173
+ graph_batch_size]
1174
else:
1175
model_executable = self.model
1176
1177
multi_modal_kwargs = model_input.multi_modal_kwargs or {}
1135
- hidden_states = model_executable(
1178
+ hidden_or_intermediate_states = model_executable(
1179
input_ids=model_input.input_tokens,
1180
positions=model_input.input_positions,
1181
kv_caches=kv_caches,
1182
attn_metadata=model_input.attn_metadata,
1183
+ intermediate_tensors=intermediate_tensors,
1184
**multi_modal_kwargs,
1185
)
1186
1143
- # Compute the logits.
1144
- logits = self.model.compute_logits(hidden_states,
1187
+ # Compute the logits in the last pipeline stage.
1188
+ if not get_pp_group().is_last_rank:
1189
+ return hidden_or_intermediate_states
1190
+
1191
+ logits = self.model.compute_logits(hidden_or_intermediate_states,
1192
model_input.sampling_metadata)
1193
1147
- # Only perform sampling in the driver worker.
1194
if not self.is_driver_worker:
1195
return []
1196
@@ -1159,9 +1205,12 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1205
assert model_input.sampling_metadata is not None
1206
indices = model_input.sampling_metadata.selected_token_indices
1207
if model_input.is_prompt:
1162
- hidden_states = hidden_states.index_select(0, indices)
1208
+ hidden_states = hidden_or_intermediate_states.index_select(
1209
+ 0, indices)
1210
elif decode_meta.use_cuda_graph:
1164
- hidden_states = hidden_states[:len(indices)]
1211
+ hidden_states = hidden_or_intermediate_states[:len(indices)]
1212
+ else:
1213
+ hidden_states = hidden_or_intermediate_states
1214
1215
output.hidden_states = hidden_states
1216
@@ -1195,13 +1244,15 @@ class CUDAGraphRunner:
1244
self,
1245
input_ids: torch.Tensor,
1246
positions: torch.Tensor,
1198
- hidden_states: Optional[torch.Tensor],
1247
+ hidden_or_intermediate_states: Optional[Union[IntermediateTensors,
1248
+ torch.Tensor]],
1249
+ intermediate_inputs: Optional[IntermediateTensors],
1250
kv_caches: List[torch.Tensor],
1251
attn_metadata: AttentionMetadata,
1252
memory_pool: Optional[Tuple[int, int]],
1253
stream: torch.cuda.Stream,
1254
**kwargs,
1204
- ) -> torch.Tensor:
1255
+ ) -> Union[torch.Tensor, IntermediateTensors]:
1256
assert self._graph is None
1257
# Run the model a few times without capturing the graph.
1258
# This is to make sure that the captured graph does not include the
@@ -1213,6 +1264,7 @@ class CUDAGraphRunner:
1264
positions,
1265
kv_caches,
1266
attn_metadata,
1267
+ intermediate_inputs,
1268
**kwargs,
1269
)
1270
torch.cuda.synchronize()
@@ -1220,18 +1272,27 @@ class CUDAGraphRunner:
1272
# Capture the graph.
1273
self._graph = torch.cuda.CUDAGraph()
1274
with torch.cuda.graph(self._graph, pool=memory_pool, stream=stream):
1223
- output_hidden_states = self.model(
1275
+ output_hidden_or_intermediate_states = self.model(
1276
input_ids,
1277
positions,
1278
kv_caches,
1279
attn_metadata,
1280
+ intermediate_inputs,
1281
**kwargs,
1282
)
1230
- if hidden_states is not None:
1231
- hidden_states.copy_(output_hidden_states)
1283
+ if hidden_or_intermediate_states is not None:
1284
+ if get_pp_group().is_last_rank:
1285
+ hidden_or_intermediate_states.copy_(
1286
+ output_hidden_or_intermediate_states)
1287
+ else:
1288
+ for key in hidden_or_intermediate_states.tensors:
1289
+ hidden_or_intermediate_states[key].copy_(
1290
+ output_hidden_or_intermediate_states[key])
1291
else:
1233
- hidden_states = output_hidden_states
1234
- del output_hidden_states
1292
+ hidden_or_intermediate_states = (
1293
+ output_hidden_or_intermediate_states)
1294
+
1295
+ del output_hidden_or_intermediate_states
1296
# make sure `output_hidden_states` is deleted
1297
# in the graph's memory pool
1298
gc.collect()
@@ -1255,8 +1316,15 @@ class CUDAGraphRunner:
1316
attn_metadata.decode_metadata.seq_lens_tensor,
1317
"block_tables": attn_metadata.decode_metadata.block_tables,
1318
}
1258
- self.output_buffers = {"hidden_states": hidden_states}
1259
- return hidden_states
1319
+ if intermediate_inputs is not None:
1320
+ self.input_buffers.update(intermediate_inputs.tensors)
1321
+ if get_pp_group().is_last_rank:
1322
+ self.output_buffers = {
1323
+ "hidden_states": hidden_or_intermediate_states
1324
+ }
1325
+ else:
1326
+ self.output_buffers = hidden_or_intermediate_states
1327
+ return hidden_or_intermediate_states
1328
1329
def forward(
1330
self,
@@ -1264,6 +1332,7 @@ class CUDAGraphRunner:
1332
positions: torch.Tensor,
1333
kv_caches: List[torch.Tensor],
1334
attn_metadata: AttentionMetadata,
1335
+ intermediate_tensors: Optional[IntermediateTensors],
1336
**kwargs,
1337
) -> torch.Tensor:
1338
# KV caches are fixed tensors, so we don't need to copy them.
@@ -1280,11 +1349,18 @@ class CUDAGraphRunner:
1349
non_blocking=True)
1350
self.input_buffers["block_tables"].copy_(
1351
attn_metadata.decode_metadata.block_tables, non_blocking=True)
1352
+ if intermediate_tensors is not None:
1353
+ for key in intermediate_tensors.tensors:
1354
+ self.input_buffers[key].copy_(intermediate_tensors[key],
1355
+ non_blocking=True)
1356
# Run the graph.
1357
self.graph.replay()
1358
1359
# Return the output tensor.
1287
- return self.output_buffers["hidden_states"]
1360
+ if get_pp_group().is_last_rank:
1361
+ return self.output_buffers["hidden_states"]
1362
+
1363
+ return self.output_buffers
1364
1365
def __call__(self, *args, **kwargs):
1366
return self.forward(*args, **kwargs)
vllm/worker/model_runner_base.py
+4
-1
@@ -5,7 +5,8 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Type,
5
6
import torch
7
8
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
8
+from vllm.sequence import (IntermediateTensors, SamplerOutput,
9
+ SequenceGroupMetadata)
10
11
if TYPE_CHECKING:
12
from vllm.attention import AttentionMetadata
@@ -137,6 +138,7 @@ class ModelRunnerBase(ABC, Generic[T]):
138
def prepare_model_input(
139
self,
140
seq_group_metadata_list: List[SequenceGroupMetadata],
141
+ virtual_engine: int = 0,
142
) -> T:
143
"""
144
Prepare the inputs to ModelRunnerBase.execute_model from an execution
@@ -150,6 +152,7 @@ class ModelRunnerBase(ABC, Generic[T]):
152
self,
153
model_input: T,
154
kv_caches: Optional[List[torch.Tensor]],
155
+ intermediate_tensors: Optional[IntermediateTensors],
156
num_steps: int = 1,
157
) -> Optional[List[SamplerOutput]]:
158
"""
vllm/worker/neuron_model_runner.py
+4
-1
@@ -9,7 +9,8 @@ from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
9
from vllm.logger import init_logger
10
from vllm.model_executor import SamplingMetadata
11
from vllm.model_executor.model_loader.neuron import get_neuron_model
12
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
12
+from vllm.sequence import (IntermediateTensors, SamplerOutput,
13
+ SequenceGroupMetadata)
14
from vllm.utils import is_pin_memory_available, make_tensor_with_pad
15
from vllm.worker.model_runner_base import ModelRunnerBase, ModelRunnerInputBase
16
@@ -175,6 +176,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
176
def prepare_model_input(
177
self,
178
seq_group_metadata_list: List[SequenceGroupMetadata],
179
+ virtual_engine: int = 0,
180
) -> ModelInputForNeuron:
181
# NOTE: We assume that all sequences in the group are all prompts or
182
# all decodes.
@@ -207,6 +209,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
209
self,
210
model_input: ModelInputForNeuron,
211
kv_caches: Optional[List[torch.Tensor]] = None,
212
+ intermediate_tensors: Optional[IntermediateTensors] = None,
213
num_steps: int = 1,
214
) -> Optional[List[SamplerOutput]]:
215
if num_steps > 1:
vllm/worker/neuron_worker.py
+1
-1
@@ -80,7 +80,7 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
80
return False
81
82
@property
83
- def kv_cache(self) -> Optional[List[torch.Tensor]]:
83
+ def kv_cache(self) -> Optional[List[List[torch.Tensor]]]:
84
return None
85
86
@torch.inference_mode()
vllm/worker/worker.py
+23
-13
@@ -59,9 +59,9 @@ class Worker(LocalOrDistributedWorkerBase):
59
self.lora_config = lora_config
60
self.load_config = load_config
61
self.is_driver_worker = is_driver_worker
62
- if self.is_driver_worker:
63
- assert self.rank == 0, "The driver worker must have rank 0."
64
-
62
+ if parallel_config and is_driver_worker:
63
+ assert rank % parallel_config.tensor_parallel_size == 0, \
64
+ "Driver worker should be rank 0 of tensor parallel group."
65
if self.model_config.trust_remote_code:
66
# note: lazy import to avoid importing torch before initializing
67
from vllm.utils import init_cached_hf_modules
@@ -99,9 +99,9 @@ class Worker(LocalOrDistributedWorkerBase):
99
)
100
# Uninitialized cache engine. Will be initialized by
101
# initialize_cache.
102
- self.cache_engine: CacheEngine
102
+ self.cache_engine: List[CacheEngine]
103
# Initialize gpu_cache as embedding models don't initialize kv_caches
104
- self.gpu_cache: Optional[List[torch.tensor]] = None
104
+ self.gpu_cache: Optional[List[List[torch.tensor]]] = None
105
106
def init_device(self) -> None:
107
if self.device_config.device.type == "cuda":
@@ -217,10 +217,15 @@ class Worker(LocalOrDistributedWorkerBase):
217
218
def _init_cache_engine(self):
219
assert self.cache_config.num_gpu_blocks is not None
220
- self.cache_engine = CacheEngine(self.cache_config, self.model_config,
221
- self.parallel_config,
222
- self.device_config)
223
- self.gpu_cache = self.cache_engine.gpu_cache
220
+ self.cache_engine = [
221
+ CacheEngine(self.cache_config, self.model_config,
222
+ self.parallel_config, self.device_config)
223
+ for _ in range(self.parallel_config.pipeline_parallel_size)
224
+ ]
225
+ self.gpu_cache = [
226
+ self.cache_engine[ve].gpu_cache
227
+ for ve in range(self.parallel_config.pipeline_parallel_size)
228
+ ]
229
230
def _warm_up_model(self) -> None:
231
if not self.model_config.enforce_eager:
@@ -234,12 +239,13 @@ class Worker(LocalOrDistributedWorkerBase):
239
return self.parallel_config.tensor_parallel_size > 1
240
241
@property
237
- def kv_cache(self) -> Optional[List[torch.Tensor]]:
242
+ def kv_cache(self) -> Optional[List[List[torch.Tensor]]]:
243
return self.gpu_cache
244
245
@torch.inference_mode()
246
def prepare_worker_input(
247
self, execute_model_req: ExecuteModelRequest) -> WorkerInput:
248
+ virtual_engine = execute_model_req.virtual_engine
249
num_seq_groups = len(execute_model_req.seq_group_metadata_list)
250
# `blocks_to_swap_in` and `blocks_to_swap_out` are cpu tensors.
251
# they contain parameters to launch cudamemcpyasync.
@@ -261,20 +267,24 @@ class Worker(LocalOrDistributedWorkerBase):
267
blocks_to_swap_in=blocks_to_swap_in,
268
blocks_to_swap_out=blocks_to_swap_out,
269
blocks_to_copy=blocks_to_copy,
270
+ virtual_engine=virtual_engine,
271
)
272
273
@torch.inference_mode()
274
def execute_worker(self, worker_input: WorkerInput) -> None:
275
+ virtual_engine = worker_input.virtual_engine
276
# Issue cache operations.
277
if (worker_input.blocks_to_swap_in is not None
278
and worker_input.blocks_to_swap_in.numel() > 0):
271
- self.cache_engine.swap_in(worker_input.blocks_to_swap_in)
279
+ self.cache_engine[virtual_engine].swap_in(
280
+ worker_input.blocks_to_swap_in)
281
if (worker_input.blocks_to_swap_out is not None
282
and worker_input.blocks_to_swap_out.numel() > 0):
274
- self.cache_engine.swap_out(worker_input.blocks_to_swap_out)
283
+ self.cache_engine[virtual_engine].swap_out(
284
+ worker_input.blocks_to_swap_out)
285
if (worker_input.blocks_to_copy is not None
286
and worker_input.blocks_to_copy.numel() > 0):
277
- self.cache_engine.copy(worker_input.blocks_to_copy)
287
+ self.cache_engine[virtual_engine].copy(worker_input.blocks_to_copy)
288
289
def add_lora(self, lora_request: LoRARequest) -> bool:
290
return self.model_runner.add_lora(lora_request)
vllm/worker/worker_base.py
+31
-9
@@ -6,10 +6,11 @@ from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union
6
7
import torch
8
9
-from vllm.distributed import broadcast_tensor_dict
9
+from vllm.distributed import broadcast_tensor_dict, get_pp_group
10
from vllm.logger import init_logger
11
from vllm.lora.request import LoRARequest
12
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
12
+from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
13
+ SamplerOutput)
14
from vllm.utils import (enable_trace_function_call_for_thread, is_hip,
15
update_environment_variables)
16
from vllm.worker.model_runner_base import ModelRunnerBase, ModelRunnerInputBase
@@ -124,6 +125,7 @@ class WorkerInput:
125
blocks_to_swap_in: Optional[torch.Tensor] = None
126
blocks_to_swap_out: Optional[torch.Tensor] = None
127
blocks_to_copy: Optional[torch.Tensor] = None
128
+ virtual_engine: int = 0
129
130
@classmethod
131
def from_broadcasted_tensor_dict(
@@ -139,6 +141,7 @@ class WorkerInput:
141
blocks_to_swap_in=tensor_dict.pop("blocks_to_swap_in"),
142
blocks_to_swap_out=tensor_dict.pop("blocks_to_swap_out"),
143
blocks_to_copy=tensor_dict.pop("blocks_to_copy"),
144
+ virtual_engine=tensor_dict["virtual_engine"],
145
)
146
147
def as_broadcastable_tensor_dict(
@@ -151,6 +154,7 @@ class WorkerInput:
154
"blocks_to_swap_in": self.blocks_to_swap_in,
155
"blocks_to_swap_out": self.blocks_to_swap_out,
156
"blocks_to_copy": self.blocks_to_copy,
157
+ "virtual_engine": self.virtual_engine,
158
}
159
160
return tensor_dict
@@ -181,11 +185,13 @@ class LocalOrDistributedWorkerBase(WorkerBase):
185
186
@property
187
@abstractmethod
184
- def kv_cache(self) -> Optional[List[torch.Tensor]]:
188
+ def kv_cache(self) -> Optional[List[List[torch.Tensor]]]:
189
"""
186
- Get the kv cache to pass to the worker's model runner. Used by the
187
- default `execute_model`. If the worker's model runner does not follow
188
- the ModelRunnerBase interface, then inherit from WorkerBase instead.
190
+ Gets the list of kv caches to pass to the worker's model runner. Each
191
+ element in the list is a kv cache corresponding to a particular virtual
192
+ engine (PP stream). Used by the default `execute_model`. If the worker's
193
+ model runner does not follow the ModelRunnerBase interface, then inherit
194
+ from WorkerBase instead.
195
"""
196
raise NotImplementedError
197
@@ -227,7 +233,8 @@ class LocalOrDistributedWorkerBase(WorkerBase):
233
execute_model_req=execute_model_req)
234
model_input: ModelRunnerInputBase = (
235
self.model_runner.prepare_model_input(
230
- execute_model_req.seq_group_metadata_list))
236
+ execute_model_req.seq_group_metadata_list,
237
+ execute_model_req.virtual_engine))
238
num_steps = execute_model_req.num_steps
239
240
if self.do_metadata_broadcast:
@@ -255,8 +262,23 @@ class LocalOrDistributedWorkerBase(WorkerBase):
262
if worker_input.num_seq_groups == 0:
263
return []
264
258
- return self.model_runner.execute_model(model_input, self.kv_cache,
259
- num_steps)
265
+ intermediate_tensors = None
266
+ if not get_pp_group().is_first_rank:
267
+ intermediate_tensors = IntermediateTensors(
268
+ get_pp_group().recv_tensor_dict())
269
+
270
+ output = self.model_runner.execute_model(
271
+ model_input, self.kv_cache[worker_input.virtual_engine]
272
+ if self.kv_cache is not None else None, intermediate_tensors,
273
+ num_steps)
274
+
275
+ if not get_pp_group().is_last_rank:
276
+ get_pp_group().send_tensor_dict(output.tensors)
277
+ return [None]
278
+
279
+ # Worker only supports single-step execution. Wrap the output in a
280
+ # list to conform to interface.
281
+ return output
282
283
284
class WorkerWrapperBase:
vllm/worker/xpu_model_runner.py
+4
-1
@@ -12,7 +12,8 @@ from vllm.distributed import broadcast_tensor_dict
12
from vllm.logger import init_logger
13
from vllm.model_executor.model_loader import get_model
14
from vllm.sampling_params import SamplingParams
15
-from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
15
+from vllm.sequence import (IntermediateTensors, SamplerOutput, SequenceData,
16
+ SequenceGroupMetadata)
17
from vllm.utils import CudaMemoryProfiler, make_tensor_with_pad
18
from vllm.worker.model_runner import AttentionMetadata, SamplingMetadata
19
from vllm.worker.model_runner_base import (
@@ -190,6 +191,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
191
def prepare_model_input(
192
self,
193
seq_group_metadata_list: List[SequenceGroupMetadata],
194
+ virtual_engine: int = 0,
195
) -> ModelInputForXPU:
196
multi_modal_input = None
197
if self.is_driver_worker:
@@ -334,6 +336,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
336
self,
337
model_input: ModelInputForXPU,
338
kv_caches: List[torch.Tensor],
339
+ intermediate_tensors: Optional[IntermediateTensors] = None,
340
num_steps: int = 1,
341
) -> Optional[List[SamplerOutput]]:
342
if num_steps > 1:
vllm/worker/xpu_worker.py
+2
-2
@@ -85,8 +85,8 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
85
)
86
# Uninitialized cache engine. Will be initialized by
87
# initialize_cache.
88
- self.cache_engine: CacheEngine
89
- self.gpu_cache: List[torch.Tensor]
88
+ self.cache_engine: List[CacheEngine]
89
+ self.gpu_cache: Optional[List[List[torch.Tensor]]]
90
91
def init_device(self) -> None:
92
if self.device_config.device.type == "xpu" and is_xpu():