@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c5832d2ae9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Core] Pipeline Parallel Support (#4412)

Signed-off-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai>

Murali Andoorveedu committed Jul 2, 2024 at 10:58 UTC c5832d2ae9431a1672d547c232ec46b1a9051ff0
82 files changed +1100 -404
.buildkite/test-pipeline.yaml
+10
@@ -74,6 +74,16 @@ steps:
74 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
75 - pytest -v -s spec_decode/e2e/test_integration_dist_tp4.py
76
77 +- label: Pipeline Parallelism Test
78 + working_dir: "/vllm-workspace/tests"
79 + num_gpus: 4
80 + commands:
81 + - TP_SIZE=2 PP_SIZE=2 EAGER_MODE=1 CHUNKED_PREFILL=1 pytest -v -s distributed/test_pipeline_parallel.py
82 + - TP_SIZE=2 PP_SIZE=2 EAGER_MODE=1 CHUNKED_PREFILL=0 pytest -v -s distributed/test_pipeline_parallel.py
83 + - PP_SIZE=4 EAGER_MODE=1 CHUNKED_PREFILL=1 pytest -v -s distributed/test_pipeline_parallel.py
84 + - PP_SIZE=4 EAGER_MODE=1 CHUNKED_PREFILL=0 pytest -v -s distributed/test_pipeline_parallel.py
85 +
86 +
87 - label: Engine Test
88 mirror_hardwares: [amd]
89 command: pytest -v -s engine tokenization test_sequence.py test_config.py test_logger.py
tests/async_engine/test_async_llm_engine.py
+13 -1
@@ -5,6 +5,7 @@ import pytest
5 import torch
6
7 from vllm import SamplingParams
8 +from vllm.config import ParallelConfig
9 from vllm.engine.async_llm_engine import AsyncEngineArgs, AsyncLLMEngine
10
11 from ..utils import wait_for_gpu_memory_to_clear
@@ -23,8 +24,11 @@ class MockEngine:
24 self.add_request_calls = 0
25 self.abort_request_calls = 0
26 self.request_id = None
27 + # Ugly, remove dependency when possible
28 + self.parallel_config = ParallelConfig(1, 1, False)
29
27 - async def step_async(self):
30 + async def step_async(self, virtual_engine):
31 + # PP size is 1, ignore virtual engine
32 self.step_calls += 1
33 return [RequestOutput(
34 request_id=self.request_id)] if self.request_id else []
@@ -32,6 +36,9 @@ class MockEngine:
36 async def process_model_inputs_async(self, *args, **kwargs):
37 pass
38
39 + async def stop_remote_worker_execution_loop_async(self):
40 + pass
41 +
42 def generate(self, request_id):
43 self.request_id = request_id
44
@@ -41,6 +48,7 @@ class MockEngine:
48 def add_request(self, **kwargs):
49 del kwargs # Unused
50 self.add_request_calls += 1
51 + print(f'Request calls: {self.add_request_calls}')
52
53 async def add_request_async(self, **kwargs):
54 self.add_request_calls += 1
@@ -53,6 +61,9 @@ class MockEngine:
61 def has_unfinished_requests(self):
62 return self.request_id is not None
63
64 + def has_unfinished_requests_for_virtual_engine(self, virtual_engine):
65 + return self.request_id is not None
66 +
67
68 class MockAsyncLLMEngine(AsyncLLMEngine):
69
@@ -76,6 +87,7 @@ async def test_new_requests_event():
87 engine.engine.generate("2")
88 await asyncio.sleep(0)
89 await asyncio.sleep(0)
90 + await asyncio.sleep(0)
91 assert engine.engine.add_request_calls == 2
92 assert engine.engine.step_calls >= 2
93 await asyncio.sleep(0.001)
tests/async_engine/test_openapi_server_ray.py
+2 -2
@@ -4,7 +4,7 @@ import pytest
4 # and debugging.
5 import ray
6
7 -from ..utils import RemoteOpenAIServer
7 +from ..utils import VLLM_PATH, RemoteOpenAIServer
8
9 # any model with a chat template should work here
10 MODEL_NAME = "facebook/opt-125m"
@@ -12,7 +12,7 @@ MODEL_NAME = "facebook/opt-125m"
12
13 @pytest.fixture(scope="module")
14 def ray_ctx():
15 - ray.init()
15 + ray.init(runtime_env={"working_dir": VLLM_PATH})
16 yield
17 ray.shutdown()
18
tests/basic_correctness/test_preemption.py
+12 -12
@@ -56,8 +56,8 @@ def test_chunked_prefill_recompute(
56 max_num_seqs=max_num_seqs,
57 ) as vllm_model:
58 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
59 - assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
60 - ARTIFICIAL_PREEMPTION_MAX_CNT)
59 + assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
60 + < ARTIFICIAL_PREEMPTION_MAX_CNT)
61
62 for i in range(len(example_prompts)):
63 hf_output_ids, hf_output_str = hf_outputs[i]
@@ -91,10 +91,10 @@ def test_preemption(
91 disable_log_stats=False,
92 ) as vllm_model:
93 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
94 - assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
95 - ARTIFICIAL_PREEMPTION_MAX_CNT)
94 + assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
95 + < ARTIFICIAL_PREEMPTION_MAX_CNT)
96 total_preemption = (
97 - vllm_model.model.llm_engine.scheduler.num_cumulative_preemption)
97 + vllm_model.model.llm_engine.scheduler[0].num_cumulative_preemption)
98
99 check_outputs_equal(
100 outputs_0_lst=hf_outputs,
@@ -147,10 +147,10 @@ def test_swap(
147 ) as vllm_model:
148 vllm_outputs = vllm_model.generate_beam_search(example_prompts,
149 beam_width, max_tokens)
150 - assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
151 - ARTIFICIAL_PREEMPTION_MAX_CNT)
150 + assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
151 + < ARTIFICIAL_PREEMPTION_MAX_CNT)
152 total_preemption = (
153 - vllm_model.model.llm_engine.scheduler.num_cumulative_preemption)
153 + vllm_model.model.llm_engine.scheduler[0].num_cumulative_preemption)
154
155 for i in range(len(example_prompts)):
156 hf_output_ids, _ = hf_outputs[i]
@@ -214,8 +214,8 @@ def test_swap_infeasible(
214 example_prompts,
215 sampling_params=sampling_params,
216 )
217 - assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
218 - ARTIFICIAL_PREEMPTION_MAX_CNT)
217 + assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
218 + < ARTIFICIAL_PREEMPTION_MAX_CNT)
219
220 # Verify the request is ignored and not hang.
221 assert req_outputs[0].outputs[0].finish_reason == "length"
@@ -252,8 +252,8 @@ def test_preemption_infeasible(
252 sampling_params=sampling_params,
253 )
254
255 - assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
256 - ARTIFICIAL_PREEMPTION_MAX_CNT)
255 + assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
256 + < ARTIFICIAL_PREEMPTION_MAX_CNT)
257
258 # Verify the request is ignored and not hang.
259 for req_output in req_outputs:
tests/distributed/test_comm_ops.py
+17 -3
@@ -32,7 +32,7 @@ def all_reduce_test_worker(tp_size: int, pp_size: int, rank: int,
32 (r + 1) for r in range(tp_size)
33 ]
34 expected = torch.sum(torch.stack(all_tensors, dim=0), dim=0)
35 - t = all_tensors[rank]
35 + t = all_tensors[rank % tp_size]
36 t = tensor_model_parallel_all_reduce(t)
37 assert torch.allclose(t, expected)
38
@@ -60,7 +60,7 @@ def all_gather_test_worker(tp_size: int, pp_size: int, rank: int,
60 for r in range(tp_size)
61 ]
62 expected = torch.cat(all_tensors, dim=all_gather_dimension)
63 - t = all_tensors[rank]
63 + t = all_tensors[rank % tp_size]
64 t = tensor_model_parallel_all_gather(t, all_gather_dimension)
65 assert torch.allclose(t, expected)
66
@@ -91,7 +91,7 @@ def broadcast_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
91 "f": torch.tensor([], dtype=torch.float32, device="cuda"),
92 }
93
94 - if rank == 0:
94 + if (rank % tp_size) == 0:
95 broadcast_tensor_dict(test_dict, src=0)
96 else:
97 recv_dict = broadcast_tensor_dict(src=0)
@@ -184,3 +184,17 @@ def test_multi_process_tensor_parallel(tp_size, test_target):
184 "test_target", [send_recv_test_worker, send_recv_tensor_dict_test_worker])
185 def test_multi_process_pipeline_parallel(pp_size, test_target):
186 multi_process_parallel(1, pp_size, test_target)
187 +
188 +
189 +@pytest.mark.skipif(torch.cuda.device_count() < 4,
190 + reason="Need at least 4 GPUs to run the test.")
191 +@pytest.mark.parametrize("tp_size", [2])
192 +@pytest.mark.parametrize("pp_size", [2])
193 +@pytest.mark.parametrize("test_target", [
194 + send_recv_test_worker, send_recv_tensor_dict_test_worker,
195 + all_reduce_test_worker, all_gather_test_worker,
196 + broadcast_tensor_dict_test_worker
197 +])
198 +def test_multi_process_tensor_parallel_pipeline_parallel(
199 + tp_size, pp_size, test_target):
200 + multi_process_parallel(tp_size, pp_size, test_target)
tests/distributed/test_pipeline_parallel.py new
+149
@@ -0,0 +1,149 @@
1 +import os
2 +
3 +import openai # use the official client for correctness check
4 +import pytest
5 +# using Ray for overall ease of process management, parallel requests,
6 +# and debugging.
7 +import ray
8 +
9 +from ..utils import VLLM_PATH, RemoteOpenAIServer
10 +
11 +# downloading lora to test lora requests
12 +
13 +# any model with a chat template should work here
14 +MODEL_NAME = "meta-llama/Meta-Llama-3-8B"
15 +EAGER_MODE = bool(int(os.getenv("EAGER_MODE", 0)))
16 +CHUNKED_PREFILL = bool(int(os.getenv("CHUNKED_PREFILL", 0)))
17 +TP_SIZE = int(os.getenv("TP_SIZE", 1))
18 +PP_SIZE = int(os.getenv("PP_SIZE", 1))
19 +
20 +pytestmark = pytest.mark.asyncio
21 +
22 +
23 +@pytest.fixture(scope="module")
24 +def ray_ctx():
25 + ray.init(runtime_env={"working_dir": VLLM_PATH})
26 + yield
27 + ray.shutdown()
28 +
29 +
30 +@pytest.fixture(scope="module")
31 +def server(ray_ctx):
32 + args = [
33 + "--model",
34 + MODEL_NAME,
35 + # use half precision for speed and memory savings in CI environment
36 + "--dtype",
37 + "bfloat16",
38 + "--pipeline-parallel-size",
39 + str(PP_SIZE),
40 + "--tensor-parallel-size",
41 + str(TP_SIZE),
42 + "--distributed-executor-backend",
43 + "ray",
44 + ]
45 + if CHUNKED_PREFILL:
46 + args += [
47 + "--enable-chunked-prefill",
48 + ]
49 + if EAGER_MODE:
50 + args += [
51 + "--enforce-eager",
52 + ]
53 + return RemoteOpenAIServer(args, num_gpus=PP_SIZE * TP_SIZE)
54 +
55 +
56 +@pytest.fixture(scope="module")
57 +def client(server):
58 + return server.get_async_client()
59 +
60 +
61 +async def test_check_models(server, client: openai.AsyncOpenAI):
62 + models = await client.models.list()
63 + models = models.data
64 + served_model = models[0]
65 + assert served_model.id == MODEL_NAME
66 + assert all(model.root == MODEL_NAME for model in models)
67 +
68 +
69 +@pytest.mark.parametrize(
70 + "model_name",
71 + [MODEL_NAME],
72 +)
73 +async def test_single_completion(server, client: openai.AsyncOpenAI,
74 + model_name: str):
75 + completion = await client.completions.create(model=model_name,
76 + prompt="Hello, my name is",
77 + max_tokens=5,
78 + temperature=0.0)
79 +
80 + assert completion.id is not None
81 + assert completion.choices is not None and len(completion.choices) == 1
82 + assert completion.choices[0].text is not None and len(
83 + completion.choices[0].text) >= 5
84 + assert completion.choices[0].finish_reason == "length"
85 + assert completion.usage == openai.types.CompletionUsage(
86 + completion_tokens=5, prompt_tokens=6, total_tokens=11)
87 +
88 + # test using token IDs
89 + completion = await client.completions.create(
90 + model=MODEL_NAME,
91 + prompt=[0, 0, 0, 0, 0],
92 + max_tokens=5,
93 + temperature=0.0,
94 + )
95 + assert completion.choices[0].text is not None and len(
96 + completion.choices[0].text) >= 5
97 +
98 +
99 +@pytest.mark.parametrize(
100 + # just test 1 lora hereafter
101 + "model_name",
102 + [MODEL_NAME],
103 +)
104 +async def test_batch_completions(server, client: openai.AsyncOpenAI,
105 + model_name: str):
106 + # test simple list
107 + batch = await client.completions.create(
108 + model=model_name,
109 + prompt=["Hello, my name is", "Hello, my name is"],
110 + max_tokens=5,
111 + temperature=0.0,
112 + )
113 + assert len(batch.choices) == 2
114 + assert batch.choices[0].text == batch.choices[1].text
115 +
116 + # test n = 2
117 + batch = await client.completions.create(
118 + model=model_name,
119 + prompt=["Hello, my name is", "Hello, my name is"],
120 + n=2,
121 + max_tokens=5,
122 + temperature=0.0,
123 + extra_body=dict(
124 + # NOTE: this has to be true for n > 1 in vLLM, but not necessary
125 + # for official client.
126 + use_beam_search=True),
127 + )
128 + assert len(batch.choices) == 4
129 + assert batch.choices[0].text != batch.choices[
130 + 1].text, "beam search should be different"
131 + assert batch.choices[0].text == batch.choices[
132 + 2].text, "two copies of the same prompt should be the same"
133 + assert batch.choices[1].text == batch.choices[
134 + 3].text, "two copies of the same prompt should be the same"
135 +
136 + # test streaming
137 + batch = await client.completions.create(
138 + model=model_name,
139 + prompt=["Hello, my name is", "Hello, my name is"],
140 + max_tokens=5,
141 + temperature=0.0,
142 + stream=True,
143 + )
144 + texts = [""] * 2
145 + async for chunk in batch:
146 + assert len(chunk.choices) == 1
147 + choice = chunk.choices[0]
148 + texts[choice.index] += choice.text
149 + assert texts[0] == texts[1]
tests/engine/output_processor/test_multi_step.py
+4 -4
@@ -32,7 +32,7 @@ def test_appends_token_ids(num_new_tokens: int, seq_output_len: int):
32
33 output_processor = MultiStepOutputProcessor(
34 detokenizer=detokenizer,
35 - scheduler=scheduler,
35 + scheduler=[scheduler],
36 seq_counter=seq_counter,
37 get_tokenizer_for_seq=lambda _: mock_tokenizer(),
38 stop_checker=stop_checker,
@@ -86,7 +86,7 @@ def test_respects_max_tokens(num_new_tokens: int, seq_prompt_len: int,
86
87 output_processor = MultiStepOutputProcessor(
88 detokenizer=detokenizer,
89 - scheduler=scheduler,
89 + scheduler=[scheduler],
90 seq_counter=seq_counter,
91 get_tokenizer_for_seq=lambda _: mock_tokenizer(),
92 stop_checker=stop_checker,
@@ -148,7 +148,7 @@ def test_respects_eos_token_id(num_new_tokens: int, seq_prompt_len: int,
148
149 output_processor = MultiStepOutputProcessor(
150 detokenizer=detokenizer,
151 - scheduler=scheduler,
151 + scheduler=[scheduler],
152 seq_counter=seq_counter,
153 get_tokenizer_for_seq=lambda _: mock_tokenizer(eos_token_id),
154 stop_checker=stop_checker,
@@ -215,7 +215,7 @@ def test_ignores_eos_token_id(num_new_tokens: int, seq_prompt_len: int,
215
216 output_processor = MultiStepOutputProcessor(
217 detokenizer=detokenizer,
218 - scheduler=scheduler,
218 + scheduler=[scheduler],
219 seq_counter=seq_counter,
220 get_tokenizer_for_seq=lambda _: mock_tokenizer(eos_token_id),
221 stop_checker=stop_checker,
tests/entrypoints/openai/test_chat.py
+2 -2
@@ -14,7 +14,7 @@ import torch
14 from huggingface_hub import snapshot_download
15 from openai import BadRequestError
16
17 -from ...utils import RemoteOpenAIServer
17 +from ...utils import VLLM_PATH, RemoteOpenAIServer
18
19 # any model with a chat template should work here
20 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
@@ -77,7 +77,7 @@ def zephyr_lora_files():
77
78 @pytest.fixture(scope="module")
79 def ray_ctx():
80 - ray.init()
80 + ray.init(runtime_env={"working_dir": VLLM_PATH})
81 yield
82 ray.shutdown()
83
tests/entrypoints/openai/test_completion.py
+2 -2
@@ -16,7 +16,7 @@ from openai import BadRequestError
16
17 from vllm.transformers_utils.tokenizer import get_tokenizer
18
19 -from ...utils import RemoteOpenAIServer
19 +from ...utils import VLLM_PATH, RemoteOpenAIServer
20
21 # any model with a chat template should work here
22 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
@@ -79,7 +79,7 @@ def zephyr_lora_files():
79
80 @pytest.fixture(scope="module")
81 def ray_ctx():
82 - ray.init()
82 + ray.init(runtime_env={"working_dir": VLLM_PATH})
83 yield
84 ray.shutdown()
85
tests/entrypoints/openai/test_embedding.py
+2 -2
@@ -5,14 +5,14 @@ import openai
5 import pytest
6 import ray
7
8 -from ...utils import RemoteOpenAIServer
8 +from ...utils import VLLM_PATH, RemoteOpenAIServer
9
10 EMBEDDING_MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
11
12
13 @pytest.fixture(scope="module")
14 def ray_ctx():
15 - ray.init()
15 + ray.init(runtime_env={"working_dir": VLLM_PATH})
16 yield
17 ray.shutdown()
18
tests/entrypoints/openai/test_models.py
+2 -2
@@ -6,7 +6,7 @@ import ray
6 # downloading lora to test lora requests
7 from huggingface_hub import snapshot_download
8
9 -from ...utils import RemoteOpenAIServer
9 +from ...utils import VLLM_PATH, RemoteOpenAIServer
10
11 # any model with a chat template should work here
12 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
@@ -22,7 +22,7 @@ def zephyr_lora_files():
22
23 @pytest.fixture(scope="module")
24 def ray_ctx():
25 - ray.init()
25 + ray.init(runtime_env={"working_dir": VLLM_PATH})
26 yield
27 ray.shutdown()
28
tests/entrypoints/openai/test_vision.py
+2 -2
@@ -24,13 +24,13 @@ TEST_IMAGE_URLS = [
24
25 @pytest.fixture(scope="module")
26 def ray_ctx():
27 - ray.init()
27 + ray.init(runtime_env={"working_dir": VLLM_PATH})
28 yield
29 ray.shutdown()
30
31
32 @pytest.fixture(scope="module")
33 -def server():
33 +def server(ray_ctx):
34 return RemoteOpenAIServer([
35 "--model",
36 MODEL_NAME,
tests/spec_decode/utils.py
+3 -3
@@ -54,9 +54,9 @@ def patch_execute_model_with_seeds(worker: Worker, rand_seeds: List[int]):
54 return new_execute_model
55
56
57 -def zero_kv_cache(cache_engine: CacheEngine):
58 - assert cache_engine.gpu_cache
59 - for key_blocks, value_blocks in cache_engine.gpu_cache:
57 +def zero_kv_cache(cache_engine: List[CacheEngine]):
58 + assert cache_engine[0].gpu_cache
59 + for key_blocks, value_blocks in cache_engine[0].gpu_cache:
60 key_blocks.zero_()
61 value_blocks.zero_()
62
tests/tensorizer_loader/test_tensorizer.py
+3 -1
@@ -22,7 +22,7 @@ from vllm.model_executor.model_loader.tensorizer import (TensorizerConfig,
22 tensorize_vllm_model)
23
24 from ..conftest import VllmRunner, cleanup
25 -from ..utils import RemoteOpenAIServer
25 +from ..utils import VLLM_PATH, RemoteOpenAIServer
26
27 # yapf conflicts with isort for this docstring
28
@@ -220,6 +220,8 @@ def test_openai_apiserver_with_tensorizer(vllm_runner, tmp_path):
220 json.dumps(model_loader_extra_config),
221 ]
222
223 + ray.init(runtime_env={"working_dir": VLLM_PATH})
224 +
225 server = RemoteOpenAIServer(openai_args)
226 print("Server ready.")
227
tests/utils.py
+10 -6
@@ -49,7 +49,6 @@ class RemoteOpenAIServer:
49 DUMMY_API_KEY = "token-abc123" # vLLM's OpenAI server does not need API key
50 MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
51
52 - @ray.remote(num_gpus=1)
52 class _RemoteRunner:
53
54 def __init__(self, cli_args: List[str], *, wait_url: str,
@@ -92,7 +91,11 @@ class RemoteOpenAIServer:
91 if hasattr(self, "proc"):
92 self.proc.terminate()
93
95 - def __init__(self, cli_args: List[str], *, auto_port: bool = True) -> None:
94 + def __init__(self,
95 + cli_args: List[str],
96 + *,
97 + auto_port: bool = True,
98 + num_gpus: int = 1) -> None:
99 if auto_port:
100 if "-p" in cli_args or "--port" in cli_args:
101 raise ValueError("You have manually specified the port"
@@ -105,10 +108,11 @@ class RemoteOpenAIServer:
108 self.host = str(args.host or 'localhost')
109 self.port = int(args.port)
110
108 - self._runner = self._RemoteRunner.remote( # type: ignore
109 - cli_args,
110 - wait_url=self.url_for("health"),
111 - wait_timeout=self.MAX_SERVER_START_WAIT_S)
111 + self._runner = ray.remote(num_gpus=num_gpus)(
112 + self._RemoteRunner).remote(
113 + cli_args,
114 + wait_url=self.url_for("health"),
115 + wait_timeout=self.MAX_SERVER_START_WAIT_S)
116
117 self._wait_until_ready()
118
tests/worker/test_swap.py
+2 -2
@@ -39,8 +39,8 @@ def test_swap() -> None:
39 num_cpu_blocks=engine_config.cache_config.num_cpu_blocks)
40
41 # Randomly initialize the cache.
42 - gpu_cache = worker.cache_engine.gpu_cache
43 - cpu_cache = worker.cache_engine.cpu_cache
42 + gpu_cache = worker.cache_engine[0].gpu_cache
43 + cpu_cache = worker.cache_engine[0].cpu_cache
44 num_layers = len(gpu_cache)
45 for i in range(num_layers):
46 gpu_key_cache, gpu_value_cache = gpu_cache[i]
vllm/config.py
+22 -3
@@ -27,6 +27,17 @@ logger = init_logger(__name__)
27 _GB = 1 << 30
28 _EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
29
30 +_PP_SUPPORTED_MODELS = [
31 + "AquilaModel",
32 + "AquilaForCausalLM",
33 + "InternLMForCausalLM",
34 + "LlamaForCausalLM",
35 + "LLaMAForCausalLM",
36 + "MistralForCausalLM",
37 + "Phi3ForCausalLM",
38 + "GPT2LMHeadModel",
39 +]
40 +
41
42 class ModelConfig:
43 """Configuration for the model.
@@ -258,6 +269,13 @@ class ModelConfig:
269 total_num_hidden_layers = getattr(self.hf_text_config,
270 "num_hidden_layers", 0)
271 pipeline_parallel_size = parallel_config.pipeline_parallel_size
272 + architectures = getattr(self.hf_config, "architectures", [])
273 + if not all(arch in _PP_SUPPORTED_MODELS
274 + for arch in architectures) and pipeline_parallel_size > 1:
275 + raise NotImplementedError(
276 + "Pipeline parallelism is only supported for the following "
277 + f" architectures: {_PP_SUPPORTED_MODELS}.")
278 +
279 if total_num_hidden_layers % pipeline_parallel_size != 0:
280 raise ValueError(
281 f"Total number of hidden layers ({total_num_hidden_layers}) "
@@ -665,9 +683,10 @@ class ParallelConfig:
683 self._verify_args()
684
685 def _verify_args(self) -> None:
668 - if self.pipeline_parallel_size > 1:
669 - raise NotImplementedError(
670 - "Pipeline parallelism is not supported yet.")
686 + if (self.pipeline_parallel_size > 1
687 + and self.distributed_executor_backend == "mp"):
688 + raise NotImplementedError("Pipeline parallelism is not supported "
689 + "yet with multiprocessing.")
690 if self.distributed_executor_backend not in ("ray", "mp", None):
691 raise ValueError(
692 "Unrecognized distributed executor backend. Supported values "
vllm/core/block_manager_v1.py
+3
@@ -471,6 +471,9 @@ class BlockSpaceManagerV1(BlockSpaceManager):
471 def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
472 # NOTE: fork does not allocate a new physical block.
473 # Thus, it is always safe from OOM.
474 + if parent_seq.seq_id not in self.block_tables:
475 + # Parent sequence has either been freed or never existed.
476 + return
477 src_block_table = self.block_tables[parent_seq.seq_id]
478 self.block_tables[child_seq.seq_id] = src_block_table.copy()
479 # When using a sliding window, blocks will be eventually reused.
vllm/core/block_manager_v2.py
+3
@@ -317,6 +317,9 @@ class BlockSpaceManagerV2(BlockSpaceManager):
317 computed_seq_block_ids) # type: ignore
318
319 def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
320 + if parent_seq.seq_id not in self.block_tables:
321 + # Parent sequence has either been freed or never existed.
322 + return
323 src_block_table = self.block_tables[parent_seq.seq_id]
324 self.block_tables[child_seq.seq_id] = src_block_table.fork()
325
vllm/core/scheduler.py
+11 -2
@@ -256,6 +256,7 @@ class Scheduler:
256 scheduler_config: SchedulerConfig,
257 cache_config: CacheConfig,
258 lora_config: Optional[LoRAConfig],
259 + pipeline_parallel_size: int = 1,
260 ) -> None:
261 self.scheduler_config = scheduler_config
262 self.cache_config = cache_config
@@ -273,11 +274,19 @@ class Scheduler:
274 BlockSpaceManagerImpl = BlockSpaceManager.get_block_space_manager_class(
275 version)
276
277 + num_gpu_blocks = cache_config.num_gpu_blocks
278 + if num_gpu_blocks:
279 + num_gpu_blocks //= pipeline_parallel_size
280 +
281 + num_cpu_blocks = cache_config.num_cpu_blocks
282 + if num_cpu_blocks:
283 + num_cpu_blocks //= pipeline_parallel_size
284 +
285 # Create the block space manager.
286 self.block_manager = BlockSpaceManagerImpl(
287 block_size=self.cache_config.block_size,
279 - num_gpu_blocks=self.cache_config.num_gpu_blocks,
280 - num_cpu_blocks=self.cache_config.num_cpu_blocks,
288 + num_gpu_blocks=num_gpu_blocks,
289 + num_cpu_blocks=num_cpu_blocks,
290 sliding_window=self.cache_config.sliding_window,
291 enable_caching=self.cache_config.enable_prefix_caching)
292
vllm/distributed/parallel_state.py
+28 -22
@@ -416,7 +416,7 @@ class GroupCoordinator:
416
417 assert dst < self.world_size, f"Invalid dst rank ({dst})"
418
419 - assert dst != self.rank, (
419 + assert dst != self.rank_in_group, (
420 "Invalid destination rank. Destination rank is the same "
421 "as the current rank.")
422
@@ -446,7 +446,7 @@ class GroupCoordinator:
446
447 assert src < self.world_size, f"Invalid src rank ({src})"
448
449 - assert src != self.rank, (
449 + assert src != self.rank_in_group, (
450 "Invalid source rank. Source rank is the same as the current rank."
451 )
452
@@ -454,7 +454,7 @@ class GroupCoordinator:
454
455 # Receive object size
456 rank_size = torch.distributed.recv(size_tensor,
457 - src=src,
457 + src=self.ranks[src],
458 group=self.cpu_group)
459
460 # Tensor to receive serialized objects into.
@@ -464,7 +464,7 @@ class GroupCoordinator:
464 device="cpu")
465
466 rank_object = torch.distributed.recv(object_tensor,
467 - src=src,
467 + src=self.ranks[src],
468 group=self.cpu_group)
469
470 assert rank_object == rank_size, (
@@ -491,10 +491,9 @@ class GroupCoordinator:
491 group = self.device_group
492 metadata_group = self.cpu_group
493 assert src < self.world_size, f"Invalid src rank ({src})"
494 - src = self.ranks[src]
494
496 - rank = self.rank
497 - if rank == src:
495 + rank_in_group = self.rank_in_group
496 + if rank_in_group == src:
497 metadata_list: List[Tuple[Any, Any]] = []
498 assert isinstance(
499 tensor_dict,
@@ -512,13 +511,13 @@ class GroupCoordinator:
511 if tensor.is_cpu:
512 # use metadata_group for CPU tensors
513 handle = torch.distributed.broadcast(tensor,
515 - src=src,
514 + src=self.ranks[src],
515 group=metadata_group,
516 async_op=True)
517 else:
518 # use group for GPU tensors
519 handle = torch.distributed.broadcast(tensor,
521 - src=src,
520 + src=self.ranks[src],
521 group=group,
522 async_op=True)
523 async_handles.append(handle)
@@ -542,15 +541,16 @@ class GroupCoordinator:
541 # use metadata_group for CPU tensors
542 handle = torch.distributed.broadcast(
543 tensor,
545 - src=src,
544 + src=self.ranks[src],
545 group=metadata_group,
546 async_op=True)
547 else:
548 # use group for GPU tensors
550 - handle = torch.distributed.broadcast(tensor,
551 - src=src,
552 - group=group,
553 - async_op=True)
549 + handle = torch.distributed.broadcast(
550 + tensor,
551 + src=self.ranks[src],
552 + group=group,
553 + async_op=True)
554 async_handles.append(handle)
555 _update_nested_dict(tensor_dict, key, tensor)
556 else:
@@ -575,7 +575,7 @@ class GroupCoordinator:
575 metadata_group = self.cpu_group
576
577 if dst is None:
578 - dst = self.next_rank
578 + dst = (self.rank_in_group + 1) % self.world_size
579 assert dst < self.world_size, f"Invalid dst rank ({dst})"
580
581 metadata_list: List[Tuple[Any, Any]] = []
@@ -593,10 +593,14 @@ class GroupCoordinator:
593 continue
594 if tensor.is_cpu:
595 # use metadata_group for CPU tensors
596 - torch.distributed.send(tensor, dst=dst, group=metadata_group)
596 + torch.distributed.send(tensor,
597 + dst=self.ranks[dst],
598 + group=metadata_group)
599 else:
600 # use group for GPU tensors
599 - torch.distributed.send(tensor, dst=dst, group=group)
601 + torch.distributed.send(tensor,
602 + dst=self.ranks[dst],
603 + group=group)
604 return None
605
606 def recv_tensor_dict(
@@ -614,7 +618,7 @@ class GroupCoordinator:
618 metadata_group = self.cpu_group
619
620 if src is None:
617 - src = self.prev_rank
621 + src = (self.rank_in_group - 1) % self.world_size
622 assert src < self.world_size, f"Invalid src rank ({src})"
623
624 recv_metadata_list = self.recv_object(src=src)
@@ -631,11 +635,13 @@ class GroupCoordinator:
635 if tensor.is_cpu:
636 # use metadata_group for CPU tensors
637 torch.distributed.recv(tensor,
634 - src=src,
638 + src=self.ranks[src],
639 group=metadata_group)
640 else:
641 # use group for GPU tensors
638 - torch.distributed.recv(tensor, src=src, group=group)
642 + torch.distributed.recv(tensor,
643 + src=self.ranks[src],
644 + group=group)
645 _update_nested_dict(tensor_dict, key, tensor)
646 else:
647 _update_nested_dict(tensor_dict, key, value)
@@ -654,7 +660,7 @@ class GroupCoordinator:
660 """Sends a tensor to the destination rank in a non-blocking way"""
661 """NOTE: `dst` is the local rank of the destination rank."""
662 if dst is None:
657 - dst = self.next_rank
663 + dst = (self.rank_in_group + 1) % self.world_size
664
665 pynccl_comm = self.pynccl_comm
666 if pynccl_comm is not None and not pynccl_comm.disabled:
@@ -669,7 +675,7 @@ class GroupCoordinator:
675 """Receives a tensor from the src rank."""
676 """NOTE: `src` is the local rank of the destination rank."""
677 if src is None:
672 - src = self.prev_rank
678 + src = (self.rank_in_group - 1) % self.world_size
679
680 tensor = torch.empty(size, dtype=dtype, device=self.device)
681 pynccl_comm = self.pynccl_comm
vllm/distributed/utils.py
+10 -1
@@ -2,7 +2,7 @@
2 # Adapted from
3 # https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/utils.py
4 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5 -from typing import Sequence
5 +from typing import Sequence, Tuple
6
7 import torch
8
@@ -46,3 +46,12 @@ def split_tensor_along_last_dim(
46 return tuple(chunk.contiguous() for chunk in tensor_list)
47
48 return tensor_list
49 +
50 +
51 +def get_pp_indices(num_hidden_layers: int, pp_rank: int,
52 + pp_size: int) -> Tuple[int, int]:
53 + layers_per_partition = divide(num_hidden_layers, pp_size)
54 + start_layer = pp_rank * layers_per_partition
55 + end_layer = start_layer + layers_per_partition
56 +
57 + return (start_layer, end_layer)
vllm/engine/async_llm_engine.py
+63 -16
@@ -211,7 +211,8 @@ class _AsyncLLMEngine(LLMEngine):
211 """Extension of LLMEngine to add async methods."""
212
213 async def step_async(
214 - self) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
214 + self, virtual_engine: int
215 + ) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
216 """Performs one decoding iteration and returns newly generated results.
217 The workers are ran asynchronously if possible.
218
@@ -221,7 +222,8 @@ class _AsyncLLMEngine(LLMEngine):
222 and updates the scheduler with the model outputs. Finally, it decodes
223 the sequences and returns the newly generated results.
224 """
224 - seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
225 + seq_group_metadata_list, scheduler_outputs = self.scheduler[
226 + virtual_engine].schedule()
227
228 if not scheduler_outputs.is_empty():
229 # Execute the model.
@@ -230,6 +232,7 @@ class _AsyncLLMEngine(LLMEngine):
232 blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
233 blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
234 blocks_to_copy=scheduler_outputs.blocks_to_copy,
235 + virtual_engine=virtual_engine,
236 num_lookahead_slots=scheduler_outputs.num_lookahead_slots,
237 running_queue_size=scheduler_outputs.running_queue_size,
238 )
@@ -248,16 +251,12 @@ class _AsyncLLMEngine(LLMEngine):
251 # Tracing
252 self.do_tracing(scheduler_outputs)
253
251 - if not request_outputs:
252 - # Stop the execute model loop in parallel workers until there are
253 - # more requests to process. This avoids waiting indefinitely in
254 - # torch.distributed ops which may otherwise timeout, and unblocks
255 - # the RPC thread in the workers so that they can process any other
256 - # queued control plane messages, such as add/remove lora adapters.
257 - await self.model_executor.stop_remote_worker_execution_loop_async()
258 -
254 return request_outputs
255
256 + async def stop_remote_worker_execution_loop_async(self) -> None:
257 + """Stop the remote worker execution loop."""
258 + await self.model_executor.stop_remote_worker_execution_loop_async()
259 +
260 async def process_model_inputs_async(
261 self,
262 request_id: str,
@@ -491,7 +490,8 @@ class AsyncLLMEngine:
490 # order of the arguments.
491 cache_config = kwargs["cache_config"]
492 parallel_config = kwargs["parallel_config"]
494 - if parallel_config.tensor_parallel_size == 1:
493 + if (parallel_config.tensor_parallel_size == 1
494 + and parallel_config.pipeline_parallel_size == 1):
495 num_gpus = cache_config.gpu_memory_utilization
496 else:
497 num_gpus = 1
@@ -499,7 +499,7 @@ class AsyncLLMEngine:
499 self._engine_class).remote
500 return engine_class(*args, **kwargs)
501
502 - async def engine_step(self) -> bool:
502 + async def engine_step(self, virtual_engine: int) -> bool:
503 """Kick the engine to process the waiting requests.
504
505 Returns True if there are in-progress requests."""
@@ -530,7 +530,7 @@ class AsyncLLMEngine:
530 if self.engine_use_ray:
531 request_outputs = await self.engine.step.remote() # type: ignore
532 else:
533 - request_outputs = await self.engine.step_async()
533 + request_outputs = await self.engine.step_async(virtual_engine)
534
535 # Put the outputs into the corresponding streams.
536 for request_output in request_outputs:
@@ -546,18 +546,65 @@ class AsyncLLMEngine:
546 self.engine.abort_request(request_ids)
547
548 async def run_engine_loop(self):
549 - has_requests_in_progress = False
549 + if self.engine_use_ray:
550 + pipeline_parallel_size = 1 # type: ignore
551 + else:
552 + pipeline_parallel_size = \
553 + self.engine.parallel_config.pipeline_parallel_size
554 + has_requests_in_progress = [False] * pipeline_parallel_size
555 while True:
551 - if not has_requests_in_progress:
556 + if not any(has_requests_in_progress):
557 logger.debug("Waiting for new requests...")
558 + # Stop the execute model loop in parallel workers until there
559 + # are more requests to process. This avoids waiting
560 + # indefinitely in torch.distributed ops which may otherwise
561 + # timeout, and unblocks the RPC thread in the workers so that
562 + # they can process any other queued control plane messages,
563 + # such as add/remove lora adapters.
564 + if self.engine_use_ray:
565 + await (self.engine.stop_remote_worker_execution_loop.
566 + remote() # type: ignore
567 + )
568 + else:
569 + await self.engine.stop_remote_worker_execution_loop_async()
570 await self._request_tracker.wait_for_new_requests()
571 logger.debug("Got new requests!")
572 + requests_in_progress = [
573 + asyncio.create_task(self.engine_step(ve))
574 + for ve in range(pipeline_parallel_size)
575 + ]
576 + has_requests_in_progress = [True] * pipeline_parallel_size
577
578 # Abort if iteration takes too long due to unrecoverable errors
579 # (eg. NCCL timeouts).
580 try:
581 async with asyncio_timeout(ENGINE_ITERATION_TIMEOUT_S):
560 - has_requests_in_progress = await self.engine_step()
582 + done, _ = await asyncio.wait(
583 + requests_in_progress,
584 + return_when=asyncio.FIRST_COMPLETED)
585 + for _ in range(pipeline_parallel_size):
586 + await asyncio.sleep(0)
587 + for task in done:
588 + result = task.result()
589 + virtual_engine = requests_in_progress.index(task)
590 + if self.engine_use_ray:
591 + has_unfinished_requests = (
592 + await (self.engine.
593 + has_unfinished_requests_for_virtual_engine.
594 + remote( # type: ignore
595 + virtual_engine)))
596 + else:
597 + has_unfinished_requests = (
598 + self.engine.
599 + has_unfinished_requests_for_virtual_engine(
600 + virtual_engine))
601 + if result or has_unfinished_requests:
602 + requests_in_progress[virtual_engine] = (
603 + asyncio.create_task(
604 + self.engine_step(virtual_engine)))
605 + has_requests_in_progress[virtual_engine] = True
606 + else:
607 + has_requests_in_progress[virtual_engine] = False
608 except asyncio.TimeoutError as exc:
609 logger.error(
610 "Engine iteration timed out. This should never happen!")
vllm/engine/llm_engine.py
+50 -15
@@ -173,6 +173,7 @@ class LLMEngine:
173 "rope_scaling=%r, rope_theta=%r, tokenizer_revision=%s, "
174 "trust_remote_code=%s, dtype=%s, max_seq_len=%d, "
175 "download_dir=%r, load_format=%s, tensor_parallel_size=%d, "
176 + "pipeline_parallel_size=%d, "
177 "disable_custom_all_reduce=%s, quantization=%s, "
178 "enforce_eager=%s, kv_cache_dtype=%s, "
179 "quantization_param_path=%s, device_config=%s, "
@@ -195,6 +196,7 @@ class LLMEngine:
196 load_config.download_dir,
197 load_config.load_format,
198 parallel_config.tensor_parallel_size,
199 + parallel_config.pipeline_parallel_size,
200 parallel_config.disable_custom_all_reduce,
201 model_config.quantization,
202 model_config.enforce_eager,
@@ -296,7 +298,11 @@ class LLMEngine:
298 # Create the scheduler.
299 # NOTE: the cache_config here have been updated with the numbers of
300 # GPU and CPU blocks, which are profiled in the distributed executor.
299 - self.scheduler = Scheduler(scheduler_config, cache_config, lora_config)
301 + self.scheduler = [
302 + Scheduler(scheduler_config, cache_config, lora_config,
303 + parallel_config.pipeline_parallel_size)
304 + for _ in range(parallel_config.pipeline_parallel_size)
305 + ]
306
307 # Metric Logging.
308 if self.log_stats:
@@ -513,8 +519,16 @@ class LLMEngine:
519 raise ValueError(
520 "Either SamplingParams or PoolingParams must be provided.")
521
516 - # Add the sequence group to the scheduler.
517 - self.scheduler.add_seq_group(seq_group)
522 + # Add the sequence group to the scheduler with least unfinished seqs.
523 + costs = [
524 + scheduler.get_num_unfinished_seq_groups()
525 + for scheduler in self.scheduler
526 + ]
527 + min_cost_scheduler = self.scheduler[costs.index(min(costs))]
528 + min_cost_scheduler.add_seq_group(seq_group)
529 +
530 + def stop_remote_worker_execution_loop(self) -> None:
531 + self.model_executor.stop_remote_worker_execution_loop()
532
533 def process_model_inputs(
534 self,
@@ -684,7 +698,8 @@ class LLMEngine:
698 >>> # abort the request
699 >>> engine.abort_request(request_id)
700 """
687 - self.scheduler.abort_seq_group(request_id)
701 + for scheduler in self.scheduler:
702 + scheduler.abort_seq_group(request_id)
703
704 def get_model_config(self) -> ModelConfig:
705 """Gets the model configuration."""
@@ -696,11 +711,20 @@ class LLMEngine:
711
712 def get_num_unfinished_requests(self) -> int:
713 """Gets the number of unfinished requests."""
699 - return self.scheduler.get_num_unfinished_seq_groups()
714 + return sum(scheduler.get_num_unfinished_seq_groups()
715 + for scheduler in self.scheduler)
716
717 def has_unfinished_requests(self) -> bool:
718 """Returns True if there are unfinished requests."""
703 - return self.scheduler.has_unfinished_seqs()
719 + return any(scheduler.has_unfinished_seqs()
720 + for scheduler in self.scheduler)
721 +
722 + def has_unfinished_requests_for_virtual_engine(
723 + self, virtual_engine: int) -> bool:
724 + """
725 + Returns True if there are unfinished requests for the virtual engine.
726 + """
727 + return self.scheduler[virtual_engine].has_unfinished_seqs()
728
729 def _process_sequence_group_outputs(
730 self,
@@ -749,7 +773,8 @@ class LLMEngine:
773 self.output_processor.process_outputs(seq_group, outputs)
774
775 # Free the finished sequence groups.
752 - self.scheduler.free_finished_seq_groups()
776 + for scheduler in self.scheduler:
777 + scheduler.free_finished_seq_groups()
778
779 # Create the outputs.
780 request_outputs: List[Union[RequestOutput,
@@ -815,7 +840,12 @@ class LLMEngine:
840 >>> if not (engine.has_unfinished_requests() or example_inputs):
841 >>> break
842 """
818 - seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
843 + if self.parallel_config.pipeline_parallel_size > 1:
844 + raise NotImplementedError(
845 + "Pipeline parallelism is only supported through AsyncLLMEngine "
846 + "as performance will be severely degraded otherwise.")
847 + seq_group_metadata_list, scheduler_outputs = self.scheduler[
848 + 0].schedule()
849
850 if not scheduler_outputs.is_empty():
851 execute_model_req = ExecuteModelRequest(
@@ -886,23 +916,28 @@ class LLMEngine:
916
917 # System State
918 # Scheduler State
889 - num_running_sys = len(self.scheduler.running)
890 - num_swapped_sys = len(self.scheduler.swapped)
891 - num_waiting_sys = len(self.scheduler.waiting)
919 + num_running_sys = sum(
920 + len(scheduler.running) for scheduler in self.scheduler)
921 + num_swapped_sys = sum(
922 + len(scheduler.swapped) for scheduler in self.scheduler)
923 + num_waiting_sys = sum(
924 + len(scheduler.waiting) for scheduler in self.scheduler)
925
926 # KV Cache Usage in %
927 num_total_gpu = self.cache_config.num_gpu_blocks
928 gpu_cache_usage_sys = 0.
929 if num_total_gpu is not None:
897 - num_free_gpu = self.scheduler.block_manager.get_num_free_gpu_blocks(
898 - )
930 + num_free_gpu = sum(
931 + scheduler.block_manager.get_num_free_gpu_blocks()
932 + for scheduler in self.scheduler)
933 gpu_cache_usage_sys = 1.0 - (num_free_gpu / num_total_gpu)
934
935 num_total_cpu = self.cache_config.num_cpu_blocks
936 cpu_cache_usage_sys = 0.
937 if num_total_cpu is not None and num_total_cpu > 0:
904 - num_free_cpu = self.scheduler.block_manager.get_num_free_cpu_blocks(
905 - )
938 + num_free_cpu = sum(
939 + scheduler.block_manager.get_num_free_cpu_blocks()
940 + for scheduler in self.scheduler)
941 cpu_cache_usage_sys = 1.0 - (num_free_cpu / num_total_cpu)
942
943 # Iteration stats
vllm/engine/output_processor/interfaces.py
+1 -1
@@ -27,7 +27,7 @@ class SequenceGroupOutputProcessor(ABC):
27 def create_output_processor(
28 scheduler_config: SchedulerConfig,
29 detokenizer: Detokenizer,
30 - scheduler: Scheduler,
30 + scheduler: List[Scheduler],
31 seq_counter: Counter,
32 get_tokenizer_for_seq: Callable[[Sequence], PreTrainedTokenizer],
33 stop_checker: "StopChecker",
vllm/engine/output_processor/multi_step.py
+3 -2
@@ -34,7 +34,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
34 def __init__(
35 self,
36 detokenizer: Detokenizer,
37 - scheduler: Scheduler,
37 + scheduler: List[Scheduler],
38 seq_counter: Counter,
39 get_tokenizer_for_seq: Callable[[Sequence], PreTrainedTokenizer],
40 stop_checker: StopChecker,
@@ -141,4 +141,5 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
141 break
142
143 if seq.is_finished():
144 - self.scheduler.free_seq(seq)
144 + for scheduler in self.scheduler:
145 + scheduler.free_seq(seq)
vllm/engine/output_processor/single_step.py
+13 -7
@@ -33,7 +33,7 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
33 self,
34 scheduler_config: SchedulerConfig,
35 detokenizer: Detokenizer,
36 - scheduler: Scheduler,
36 + scheduler: List[Scheduler],
37 seq_counter: Counter,
38 stop_checker: StopChecker,
39 ):
@@ -95,7 +95,8 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
95 # not be used in the future iterations.
96 parent.status = SequenceStatus.FINISHED_ABORTED
97 seq_group.remove(parent.seq_id)
98 - self.scheduler.free_seq(parent)
98 + for scheduler in self.scheduler:
99 + scheduler.free_seq(parent)
100 continue
101 # Fork the parent sequence if there are multiple child samples.
102 for child_sample in child_samples[:-1]:
@@ -133,7 +134,8 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
134 if seq is not parent:
135 seq_group.add(seq)
136 if not seq.is_finished():
136 - self.scheduler.fork_seq(parent, seq)
137 + for scheduler in self.scheduler:
138 + scheduler.fork_seq(parent, seq)
139
140 # Free the finished and selected parent sequences' memory in block
141 # manager. Keep them in the sequence group as candidate output.
@@ -141,7 +143,8 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
143 # old sequences.
144 for seq, parent in child_seqs:
145 if seq is parent and seq.is_finished():
144 - self.scheduler.free_seq(seq)
146 + for scheduler in self.scheduler:
147 + scheduler.free_seq(seq)
148 return
149
150 # Beam search case
@@ -226,13 +229,15 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
229 if seq is not parent:
230 seq_group.add(seq)
231 if not seq.is_finished():
229 - self.scheduler.fork_seq(parent, seq)
232 + for scheduler in self.scheduler:
233 + scheduler.fork_seq(parent, seq)
234
235 # Free the finished and selected parent sequences' memory in block
236 # manager. Keep them in the sequence group as candidate output.
237 for seq, parent in selected_child_seqs:
238 if seq is parent and seq.is_finished():
235 - self.scheduler.free_seq(seq)
239 + for scheduler in self.scheduler:
240 + scheduler.free_seq(seq)
241
242 # Remove the unselected parent sequences from the sequence group and
243 # free their memory in block manager.
@@ -241,7 +246,8 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
246 # Remove the parent sequence if it is not selected for next
247 # iteration
248 seq_group.remove(seq.seq_id)
244 - self.scheduler.free_seq(seq)
249 + for scheduler in self.scheduler:
250 + scheduler.free_seq(seq)
251
252 def _check_beam_search_early_stopping(
253 self,
vllm/executor/distributed_gpu_executor.py
+6 -6
@@ -69,7 +69,7 @@ class DistributedGPUExecutor(GPUExecutor):
69 if self.parallel_worker_tasks is None:
70 self.parallel_worker_tasks = self._run_workers(
71 "start_worker_execution_loop",
72 - async_run_remote_workers_only=True,
72 + async_run_tensor_parallel_workers_only=True,
73 **self.extra_execute_model_run_workers_kwargs)
74
75 # Only the driver worker returns the sampling results.
@@ -138,17 +138,17 @@ class DistributedGPUExecutor(GPUExecutor):
138 self,
139 method: str,
140 *args,
141 - async_run_remote_workers_only: bool = False,
141 + async_run_tensor_parallel_workers_only: bool = False,
142 max_concurrent_workers: Optional[int] = None,
143 **kwargs,
144 ) -> Any:
145 """Runs the given method on all workers.
146
147 Args:
148 - async_run_remote_workers_only: If True the method will be run only
149 - in the remote workers, not the driver worker. It will also be
150 - run asynchronously and return a list of futures rather than
151 - blocking on the results.
148 + async_run_tensor_parallel_workers_only: If True the method will be
149 + run only in the remote TP workers, not the driver worker.
150 + It will also be run asynchronously and return a list of futures
151 + rather than blocking on the results.
152 """
153 raise NotImplementedError
154
vllm/executor/executor_base.py
+25
@@ -1,3 +1,4 @@
1 +import asyncio
2 from abc import ABC, abstractmethod
3 from typing import List, Optional, Set, Tuple
4
@@ -110,6 +111,30 @@ class ExecutorBase(ABC):
111
112 class ExecutorAsyncBase(ExecutorBase):
113
114 + def __init__(
115 + self,
116 + model_config: ModelConfig,
117 + cache_config: CacheConfig,
118 + parallel_config: ParallelConfig,
119 + scheduler_config: SchedulerConfig,
120 + device_config: DeviceConfig,
121 + load_config: LoadConfig,
122 + lora_config: Optional[LoRAConfig],
123 + vision_language_config: Optional[VisionLanguageConfig],
124 + speculative_config: Optional[SpeculativeConfig],
125 + ) -> None:
126 + # This locks each pipeline parallel stage so multiple virtual engines
127 + # can't execute on the same stage at the same time
128 + self.pp_locks = [
129 + asyncio.Lock()
130 + for _ in range(parallel_config.pipeline_parallel_size)
131 + ]
132 +
133 + super().__init__(model_config, cache_config, parallel_config,
134 + scheduler_config, device_config, load_config,
135 + lora_config, vision_language_config,
136 + speculative_config)
137 +
138 @abstractmethod
139 async def execute_model_async(
140 self,
vllm/executor/gpu_executor.py
+2 -1
@@ -45,7 +45,8 @@ class GPUExecutor(ExecutorBase):
45 lora_config=self.lora_config,
46 vision_language_config=self.vision_language_config,
47 speculative_config=self.speculative_config,
48 - is_driver_worker=rank == 0,
48 + is_driver_worker=(not self.parallel_config)
49 + or (rank % self.parallel_config.tensor_parallel_size == 0),
50 )
51
52 def _create_worker(self,
vllm/executor/multiproc_gpu_executor.py
+6 -6
@@ -91,17 +91,17 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
91 self,
92 method: str,
93 *args,
94 - async_run_remote_workers_only: bool = False,
94 + async_run_tensor_parallel_workers_only: bool = False,
95 max_concurrent_workers: Optional[int] = None,
96 **kwargs,
97 ) -> Any:
98 """Runs the given method on all workers.
99
100 Args:
101 - async_run_remote_workers_only: If True the method will be run only
102 - in the remote workers, not the driver worker. It will also be
103 - run asynchronously and return a list of futures rather than
104 - blocking on the results.
101 + async_run_tensor_parallel_workers_only: If True the method will be
102 + run only in the remote TP workers, not the driver worker.
103 + It will also be run asynchronously and return a list of futures
104 + rather than blocking on the results.
105 """
106
107 if max_concurrent_workers:
@@ -114,7 +114,7 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
114 for worker in self.workers
115 ]
116
117 - if async_run_remote_workers_only:
117 + if async_run_tensor_parallel_workers_only:
118 # Just return futures
119 return worker_outputs
120
vllm/executor/ray_gpu_executor.py
+59 -12
@@ -62,7 +62,8 @@ class RayGPUExecutor(DistributedGPUExecutor):
62
63 def _init_workers_ray(self, placement_group: "PlacementGroup",
64 **ray_remote_kwargs):
65 - if self.parallel_config.tensor_parallel_size == 1:
65 + if (self.parallel_config.tensor_parallel_size == 1
66 + and self.parallel_config.pipeline_parallel_size == 1):
67 # For single GPU case, we use a ray worker with constrained memory.
68 num_gpus = self.cache_config.gpu_memory_utilization
69 else:
@@ -189,6 +190,26 @@ class RayGPUExecutor(DistributedGPUExecutor):
190 max_concurrent_workers=self.parallel_config.
191 max_parallel_loading_workers)
192
193 + # This is the list of workers that are rank 0 of each TP group EXCEPT
194 + # global rank 0. These are the workers that will broadcast to the
195 + # rest of the workers.
196 + self.tp_driver_workers: List[RayWorkerWrapper] = []
197 + # This is the list of workers that are not drivers and not the first
198 + # worker in a TP group. These are the workers that will be
199 + # broadcasted to.
200 + self.non_driver_workers: List[RayWorkerWrapper] = []
201 +
202 + for pp_rank in range(self.parallel_config.pipeline_parallel_size):
203 + for tp_rank in range(self.parallel_config.tensor_parallel_size):
204 + rank = (pp_rank *
205 + self.parallel_config.tensor_parallel_size) + tp_rank
206 + if rank == 0:
207 + pass
208 + elif rank % self.parallel_config.tensor_parallel_size == 0:
209 + self.tp_driver_workers.append(self.workers[rank - 1])
210 + else:
211 + self.non_driver_workers.append(self.workers[rank - 1])
212 +
213 def _driver_execute_model(
214 self, execute_model_req: Optional[ExecuteModelRequest]
215 ) -> Optional[List[SamplerOutput]]:
@@ -204,7 +225,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
225 self,
226 method: str,
227 *args,
207 - async_run_remote_workers_only: bool = False,
228 + async_run_tensor_parallel_workers_only: bool = False,
229 all_args: Optional[List[Tuple[Any, ...]]] = None,
230 all_kwargs: Optional[List[Dict[str, Any]]] = None,
231 use_dummy_driver: bool = False,
@@ -215,10 +236,11 @@ class RayGPUExecutor(DistributedGPUExecutor):
236 """Runs the given method on all workers. Can be used in the following
237 ways:
238
218 - - async_run_remote_workers_only: If True the method will be run only
219 - in the remote workers, not the driver worker. It will also be
220 - run asynchronously and return a list of futures rather than blocking
221 - on the results.
239 + Args:
240 + - async_run_tensor_parallel_workers_only: If True the method will be
241 + run only in the remote TP workers, not the driver worker.
242 + It will also be run asynchronously and return a list of futures
243 + rather than blocking on the results.
244 - args/kwargs: All workers share the same args/kwargs
245 - all_args/all_kwargs: args/kwargs for each worker are specified
246 individually
@@ -228,7 +250,9 @@ class RayGPUExecutor(DistributedGPUExecutor):
250 raise NotImplementedError(
251 "max_concurrent_workers is not supported yet.")
252
231 - count = len(self.workers)
253 + count = len(self.workers) if not \
254 + async_run_tensor_parallel_workers_only \
255 + else len(self.non_driver_workers)
256 all_worker_args = repeat(args, count) if all_args is None \
257 else islice(all_args, 1, None)
258 all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
@@ -242,14 +266,17 @@ class RayGPUExecutor(DistributedGPUExecutor):
266 ray_worker_outputs = []
267 else:
268 # Start the ray workers first.
269 + ray_workers = self.workers
270 + if async_run_tensor_parallel_workers_only:
271 + ray_workers = self.non_driver_workers
272 ray_worker_outputs = [
273 worker.execute_method.remote(method, *worker_args,
274 **worker_kwargs)
275 for (worker, worker_args, worker_kwargs
249 - ) in zip(self.workers, all_worker_args, all_worker_kwargs)
276 + ) in zip(ray_workers, all_worker_args, all_worker_kwargs)
277 ]
278
252 - if async_run_remote_workers_only:
279 + if async_run_tensor_parallel_workers_only:
280 # Just return futures
281 return ray_worker_outputs
282
@@ -319,12 +346,32 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
346 self,
347 execute_model_req: Optional[ExecuteModelRequest] = None
348 ) -> List[SamplerOutput]:
322 - return await self.driver_exec_method("execute_model",
323 - execute_model_req)
349 +
350 + async def _run_task_with_lock(task, lock, *args, **kwargs):
351 + async with lock:
352 + return await task(*args, **kwargs)
353 +
354 + tasks = []
355 + tasks.append(
356 + asyncio.create_task(
357 + _run_task_with_lock(self.driver_exec_method, self.pp_locks[0],
358 + "execute_model", execute_model_req)))
359 + for pp_rank, driver_worker in enumerate(self.tp_driver_workers,
360 + start=1):
361 + tasks.append(
362 + asyncio.create_task(
363 + _run_task_with_lock(driver_worker.execute_method.remote,
364 + self.pp_locks[pp_rank],
365 + "execute_model", execute_model_req)))
366 +
367 + results = await asyncio.gather(*tasks)
368 +
369 + # Only the last PP stage has the final results.
370 + return results[-1]
371
372 async def _start_worker_execution_loop(self):
373 coros = [
374 worker.execute_method.remote("start_worker_execution_loop")
328 - for worker in self.workers
375 + for worker in self.non_driver_workers
376 ]
377 return await asyncio.gather(*coros)
vllm/model_executor/models/arctic.py
+2 -1
@@ -29,7 +29,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
29 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
30 from vllm.model_executor.sampling_metadata import SamplingMetadata
31 from vllm.model_executor.utils import set_weight_attrs
32 -from vllm.sequence import SamplerOutput
32 +from vllm.sequence import IntermediateTensors, SamplerOutput
33 from vllm.transformers_utils.configs.arctic import ArcticConfig
34
35 logger = init_logger(__name__)
@@ -426,6 +426,7 @@ class ArcticForCausalLM(nn.Module):
426 positions: torch.Tensor,
427 kv_caches: List[torch.Tensor],
428 attn_metadata: AttentionMetadata,
429 + intermediate_tensors: Optional[IntermediateTensors] = None,
430 ) -> torch.Tensor:
431 hidden_states = self.model(input_ids, positions, kv_caches,
432 attn_metadata)
vllm/model_executor/models/baichuan.py
+2 -1
@@ -43,7 +43,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
43 ParallelLMHead, VocabParallelEmbedding)
44 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45 from vllm.model_executor.sampling_metadata import SamplingMetadata
46 -from vllm.sequence import SamplerOutput
46 +from vllm.sequence import IntermediateTensors, SamplerOutput
47
48 from .interfaces import SupportsLoRA
49
@@ -338,6 +338,7 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA):
338 positions: torch.Tensor,
339 kv_caches: List[torch.Tensor],
340 attn_metadata: AttentionMetadata,
341 + intermediate_tensors: Optional[IntermediateTensors] = None,
342 ) -> torch.Tensor:
343 hidden_states = self.model(input_ids, positions, kv_caches,
344 attn_metadata)
vllm/model_executor/models/bloom.py
+2 -1
@@ -39,7 +39,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
39 VocabParallelEmbedding)
40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41 from vllm.model_executor.sampling_metadata import SamplingMetadata
42 -from vllm.sequence import SamplerOutput
42 +from vllm.sequence import IntermediateTensors, SamplerOutput
43
44
45 def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
@@ -286,6 +286,7 @@ class BloomForCausalLM(nn.Module):
286 positions: torch.Tensor,
287 kv_caches: List[torch.Tensor],
288 attn_metadata: AttentionMetadata,
289 + intermediate_tensors: Optional[IntermediateTensors] = None,
290 ) -> torch.Tensor:
291 hidden_states = self.transformer(input_ids, positions, kv_caches,
292 attn_metadata)
vllm/model_executor/models/chatglm.py
+2 -1
@@ -25,7 +25,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
25 ParallelLMHead, VocabParallelEmbedding)
26 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27 from vllm.model_executor.sampling_metadata import SamplingMetadata
28 -from vllm.sequence import SamplerOutput
28 +from vllm.sequence import IntermediateTensors, SamplerOutput
29 from vllm.transformers_utils.configs import ChatGLMConfig
30
31 from .interfaces import SupportsLoRA
@@ -365,6 +365,7 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA):
365 positions: torch.Tensor,
366 kv_caches: List[torch.Tensor],
367 attn_metadata: AttentionMetadata,
368 + intermediate_tensors: Optional[IntermediateTensors] = None,
369 ) -> torch.Tensor:
370 hidden_states = self.transformer(input_ids, positions, kv_caches,
371 attn_metadata)
vllm/model_executor/models/commandr.py
+2 -1
@@ -46,7 +46,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
46 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
47 from vllm.model_executor.sampling_metadata import SamplingMetadata
48 from vllm.model_executor.utils import set_weight_attrs
49 -from vllm.sequence import SamplerOutput
49 +from vllm.sequence import IntermediateTensors, SamplerOutput
50
51
52 @torch.compile
@@ -353,6 +353,7 @@ class CohereForCausalLM(nn.Module):
353 positions: torch.Tensor,
354 kv_caches: List[torch.Tensor],
355 attn_metadata: AttentionMetadata,
356 + intermediate_tensors: Optional[IntermediateTensors] = None,
357 ) -> torch.Tensor:
358 hidden_states = self.model(input_ids, positions, kv_caches,
359 attn_metadata)
vllm/model_executor/models/dbrx.py
+2 -1
@@ -23,7 +23,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
23 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24 from vllm.model_executor.sampling_metadata import SamplingMetadata
25 from vllm.model_executor.utils import set_weight_attrs
26 -from vllm.sequence import SamplerOutput
26 +from vllm.sequence import IntermediateTensors, SamplerOutput
27 from vllm.transformers_utils.configs.dbrx import DbrxConfig
28
29
@@ -381,6 +381,7 @@ class DbrxForCausalLM(nn.Module):
381 positions: torch.Tensor,
382 kv_caches: List[torch.Tensor],
383 attn_metadata: AttentionMetadata,
384 + intermediate_tensors: Optional[IntermediateTensors] = None,
385 ) -> torch.Tensor:
386 hidden_states = self.transformer(input_ids, positions, kv_caches,
387 attn_metadata)
vllm/model_executor/models/deepseek.py
+2 -1
@@ -48,7 +48,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
48 ParallelLMHead, VocabParallelEmbedding)
49 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
50 from vllm.model_executor.sampling_metadata import SamplingMetadata
51 -from vllm.sequence import SamplerOutput
51 +from vllm.sequence import IntermediateTensors, SamplerOutput
52
53
54 class DeepseekMLP(nn.Module):
@@ -387,6 +387,7 @@ class DeepseekForCausalLM(nn.Module):
387 positions: torch.Tensor,
388 kv_caches: List[torch.Tensor],
389 attn_metadata: AttentionMetadata,
390 + intermediate_tensors: Optional[IntermediateTensors] = None,
391 ) -> torch.Tensor:
392 hidden_states = self.model(input_ids, positions, kv_caches,
393 attn_metadata)
vllm/model_executor/models/deepseek_v2.py
+2 -1
@@ -48,7 +48,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
48 ParallelLMHead, VocabParallelEmbedding)
49 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
50 from vllm.model_executor.sampling_metadata import SamplingMetadata
51 -from vllm.sequence import SamplerOutput
51 +from vllm.sequence import IntermediateTensors, SamplerOutput
52
53
54 class DeepseekV2MLP(nn.Module):
@@ -475,6 +475,7 @@ class DeepseekV2ForCausalLM(nn.Module):
475 positions: torch.Tensor,
476 kv_caches: List[torch.Tensor],
477 attn_metadata: AttentionMetadata,
478 + intermediate_tensors: Optional[IntermediateTensors] = None,
479 ) -> torch.Tensor:
480 hidden_states = self.model(input_ids, positions, kv_caches,
481 attn_metadata)
vllm/model_executor/models/falcon.py
+2 -1
@@ -44,7 +44,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
44 ParallelLMHead, VocabParallelEmbedding)
45 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
46 from vllm.model_executor.sampling_metadata import SamplingMetadata
47 -from vllm.sequence import SamplerOutput
47 +from vllm.sequence import IntermediateTensors, SamplerOutput
48 from vllm.transformers_utils.configs import RWConfig
49
50 FalconConfig = Union[HF_FalconConfig, RWConfig]
@@ -410,6 +410,7 @@ class FalconForCausalLM(nn.Module):
410 positions: torch.Tensor,
411 kv_caches: List[torch.Tensor],
412 attn_metadata: AttentionMetadata,
413 + intermediate_tensors: Optional[IntermediateTensors] = None,
414 ) -> torch.Tensor:
415 hidden_states = self.transformer(
416 input_ids,
vllm/model_executor/models/gemma.py
+2 -1
@@ -39,7 +39,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
39 VocabParallelEmbedding)
40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41 from vllm.model_executor.sampling_metadata import SamplingMetadata
42 -from vllm.sequence import SamplerOutput
42 +from vllm.sequence import IntermediateTensors, SamplerOutput
43
44 from .interfaces import SupportsLoRA
45
@@ -339,6 +339,7 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA):
339 positions: torch.Tensor,
340 kv_caches: List[torch.Tensor],
341 attn_metadata: AttentionMetadata,
342 + intermediate_tensors: Optional[IntermediateTensors] = None,
343 ) -> torch.Tensor:
344 hidden_states = self.model(input_ids, positions, kv_caches,
345 attn_metadata)
vllm/model_executor/models/gemma2.py
+2 -1
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
37 VocabParallelEmbedding)
38 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
39 from vllm.model_executor.sampling_metadata import SamplingMetadata
40 -from vllm.sequence import SamplerOutput
40 +from vllm.sequence import IntermediateTensors, SamplerOutput
41 from vllm.utils import print_warning_once
42
43 from .interfaces import SupportsLoRA
@@ -338,6 +338,7 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA):
338 positions: torch.Tensor,
339 kv_caches: List[torch.Tensor],
340 attn_metadata: AttentionMetadata,
341 + intermediate_tensors: Optional[IntermediateTensors] = None,
342 ) -> torch.Tensor:
343 hidden_states = self.model(input_ids, positions, kv_caches,
344 attn_metadata)
vllm/model_executor/models/gpt2.py
+61 -27
@@ -17,7 +17,7 @@
17 # See the License for the specific language governing permissions and
18 # limitations under the License.
19 """Inference-only GPT-2 model compatible with HuggingFace weights."""
20 -from typing import Iterable, List, Optional, Tuple
20 +from typing import Iterable, List, Optional, Tuple, Union
21
22 import torch
23 from torch import nn
@@ -25,7 +25,9 @@ from transformers import GPT2Config
25
26 from vllm.attention import Attention, AttentionMetadata
27 from vllm.config import CacheConfig
28 -from vllm.distributed import get_tensor_model_parallel_world_size
28 +from vllm.distributed.parallel_state import (
29 + get_pp_group, get_tensor_model_parallel_world_size)
30 +from vllm.distributed.utils import get_pp_indices
31 from vllm.model_executor.layers.activation import get_act_fn
32 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33 QKVParallelLinear,
@@ -38,7 +40,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
40 VocabParallelEmbedding)
41 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
42 from vllm.model_executor.sampling_metadata import SamplingMetadata
41 -from vllm.sequence import SamplerOutput
43 +from vllm.sequence import IntermediateTensors, SamplerOutput
44
45
46 class GPT2Attention(nn.Module):
@@ -181,10 +183,18 @@ class GPT2Model(nn.Module):
183 self.embed_dim = config.hidden_size
184 self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
185 self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
184 - self.h = nn.ModuleList([
185 - GPT2Block(config, cache_config, quant_config)
186 - for _ in range(config.num_hidden_layers)
187 - ])
186 + self.start_layer, self.end_layer = get_pp_indices(
187 + config.num_hidden_layers,
188 + get_pp_group().rank_in_group,
189 + get_pp_group().world_size)
190 + self.h = nn.ModuleList(
191 + [nn.Identity() for _ in range(self.start_layer)] + [
192 + GPT2Block(config, cache_config, quant_config)
193 + for _ in range(self.start_layer, self.end_layer)
194 + ] + [
195 + nn.Identity()
196 + for _ in range(self.end_layer, config.num_hidden_layers)
197 + ])
198 self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
199
200 def forward(
@@ -193,14 +203,24 @@ class GPT2Model(nn.Module):
203 position_ids: torch.Tensor,
204 kv_caches: List[torch.Tensor],
205 attn_metadata: AttentionMetadata,
196 - ) -> torch.Tensor:
197 - inputs_embeds = self.wte(input_ids)
198 - position_embeds = self.wpe(position_ids)
199 - hidden_states = inputs_embeds + position_embeds
206 + intermediate_tensors: Optional[IntermediateTensors],
207 + ) -> Union[torch.Tensor, IntermediateTensors]:
208 + if get_pp_group().is_first_rank:
209 + inputs_embeds = self.wte(input_ids)
210 + position_embeds = self.wpe(position_ids)
211 + hidden_states = inputs_embeds + position_embeds
212 + else:
213 + assert intermediate_tensors is not None
214 + hidden_states = intermediate_tensors["hidden_states"]
215
201 - for i in range(len(self.h)):
216 + for i in range(self.start_layer, self.end_layer):
217 layer = self.h[i]
203 - hidden_states = layer(hidden_states, kv_caches[i], attn_metadata)
218 + hidden_states = layer(hidden_states,
219 + kv_caches[i - self.start_layer],
220 + attn_metadata)
221 +
222 + if not get_pp_group().is_last_rank:
223 + return IntermediateTensors({"hidden_states": hidden_states})
224
225 hidden_states = self.ln_f(hidden_states)
226 return hidden_states
@@ -228,9 +248,10 @@ class GPT2LMHeadModel(nn.Module):
248 positions: torch.Tensor,
249 kv_caches: List[torch.Tensor],
250 attn_metadata: AttentionMetadata,
251 + intermediate_tensors: Optional[IntermediateTensors] = None,
252 ) -> torch.Tensor:
253 hidden_states = self.transformer(input_ids, positions, kv_caches,
233 - attn_metadata)
254 + attn_metadata, intermediate_tensors)
255 return hidden_states
256
257 def compute_logits(self, hidden_states: torch.Tensor,
@@ -247,6 +268,16 @@ class GPT2LMHeadModel(nn.Module):
268 next_tokens = self.sampler(logits, sampling_metadata)
269 return next_tokens
270
271 + def make_empty_intermediate_tensors(
272 + self, batch_size: int, dtype: torch.dtype,
273 + device: torch.device) -> IntermediateTensors:
274 + return IntermediateTensors({
275 + "hidden_states":
276 + torch.zeros((batch_size, self.config.hidden_size),
277 + dtype=dtype,
278 + device=device),
279 + })
280 +
281 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
282 params_dict = dict(self.named_parameters(remove_duplicate=False))
283 for name, loaded_weight in weights:
@@ -260,16 +291,19 @@ class GPT2LMHeadModel(nn.Module):
291 continue
292 if not name.startswith("transformer."):
293 name = "transformer." + name
263 - param = params_dict[name]
264 - # The HF's GPT-2 implementation uses Conv1D instead of Linear.
265 - # Because of this, we need to transpose the weights.
266 - # Note(zhuohan): the logic below might break quantized models.
267 - for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]:
268 - if conv1d_weight_name not in name:
269 - continue
270 - if not name.endswith(".weight"):
271 - continue
272 - loaded_weight = loaded_weight.t()
273 - weight_loader = getattr(param, "weight_loader",
274 - default_weight_loader)
275 - weight_loader(param, loaded_weight)
294 + try:
295 + param = params_dict[name]
296 + # The HF's GPT-2 implementation uses Conv1D instead of Linear.
297 + # Because of this, we need to transpose the weights.
298 + # Note(zhuohan): the logic below might break quantized models.
299 + for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]:
300 + if conv1d_weight_name not in name:
301 + continue
302 + if not name.endswith(".weight"):
303 + continue
304 + loaded_weight = loaded_weight.t()
305 + weight_loader = getattr(param, "weight_loader",
306 + default_weight_loader)
307 + weight_loader(param, loaded_weight)
308 + except KeyError:
309 + continue
vllm/model_executor/models/gpt_bigcode.py
+2 -1
@@ -39,7 +39,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
39 VocabParallelEmbedding)
40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41 from vllm.model_executor.sampling_metadata import SamplingMetadata
42 -from vllm.sequence import SamplerOutput
42 +from vllm.sequence import IntermediateTensors, SamplerOutput
43
44 from .interfaces import SupportsLoRA
45
@@ -273,6 +273,7 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA):
273 positions: torch.Tensor,
274 kv_caches: List[torch.Tensor],
275 attn_metadata: AttentionMetadata,
276 + intermediate_tensors: Optional[IntermediateTensors] = None,
277 ) -> torch.Tensor:
278 hidden_states = self.transformer(input_ids, positions, kv_caches,
279 attn_metadata)
vllm/model_executor/models/gpt_j.py
+2 -1
@@ -38,7 +38,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
38 ParallelLMHead, VocabParallelEmbedding)
39 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40 from vllm.model_executor.sampling_metadata import SamplingMetadata
41 -from vllm.sequence import SamplerOutput
41 +from vllm.sequence import IntermediateTensors, SamplerOutput
42
43
44 class GPTJAttention(nn.Module):
@@ -239,6 +239,7 @@ class GPTJForCausalLM(nn.Module):
239 positions: torch.Tensor,
240 kv_caches: List[torch.Tensor],
241 attn_metadata: AttentionMetadata,
242 + intermediate_tensors: Optional[IntermediateTensors] = None,
243 ) -> torch.Tensor:
244 hidden_states = self.transformer(input_ids, positions, kv_caches,
245 attn_metadata)
vllm/model_executor/models/gpt_neox.py
+2 -1
@@ -38,7 +38,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
38 ParallelLMHead, VocabParallelEmbedding)
39 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40 from vllm.model_executor.sampling_metadata import SamplingMetadata
41 -from vllm.sequence import SamplerOutput
41 +from vllm.sequence import IntermediateTensors, SamplerOutput
42
43
44 class GPTNeoXAttention(nn.Module):
@@ -251,6 +251,7 @@ class GPTNeoXForCausalLM(nn.Module):
251 positions: torch.Tensor,
252 kv_caches: List[torch.Tensor],
253 attn_metadata: AttentionMetadata,
254 + intermediate_tensors: Optional[IntermediateTensors] = None,
255 ) -> torch.Tensor:
256 hidden_states = self.gpt_neox(input_ids, positions, kv_caches,
257 attn_metadata)
vllm/model_executor/models/internlm2.py
+2 -1
@@ -22,7 +22,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
22 ParallelLMHead, VocabParallelEmbedding)
23 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24 from vllm.model_executor.sampling_metadata import SamplingMetadata
25 -from vllm.sequence import SamplerOutput
25 +from vllm.sequence import IntermediateTensors, SamplerOutput
26
27
28 class InternLM2MLP(nn.Module):
@@ -263,6 +263,7 @@ class InternLM2ForCausalLM(nn.Module):
263 positions: torch.Tensor,
264 kv_caches: List[torch.Tensor],
265 attn_metadata: AttentionMetadata,
266 + intermediate_tensors: IntermediateTensors,
267 ) -> torch.Tensor:
268 hidden_states = self.model(input_ids, positions, kv_caches,
269 attn_metadata)
vllm/model_executor/models/jais.py
+2 -1
@@ -40,7 +40,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
40 VocabParallelEmbedding)
41 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
42 from vllm.model_executor.sampling_metadata import SamplingMetadata
43 -from vllm.sequence import SamplerOutput
43 +from vllm.sequence import IntermediateTensors, SamplerOutput
44 from vllm.transformers_utils.configs import JAISConfig
45
46
@@ -289,6 +289,7 @@ class JAISLMHeadModel(nn.Module):
289 positions: torch.Tensor,
290 kv_caches: List[torch.Tensor],
291 attn_metadata: AttentionMetadata,
292 + intermediate_tensors: Optional[IntermediateTensors] = None,
293 ) -> torch.Tensor:
294 hidden_states = self.transformer(input_ids, positions, kv_caches,
295 attn_metadata)
vllm/model_executor/models/llama.py
+73 -28
@@ -21,7 +21,7 @@
21 # See the License for the specific language governing permissions and
22 # limitations under the License.
23 """Inference-only LLaMA model compatible with HuggingFace weights."""
24 -from typing import Any, Dict, Iterable, List, Optional, Tuple
24 +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
25
26 import torch
27 from torch import nn
@@ -29,7 +29,8 @@ from transformers import LlamaConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.config import CacheConfig, LoRAConfig
32 -from vllm.distributed import (get_tensor_model_parallel_rank,
32 +from vllm.distributed import (get_pp_group, get_pp_indices,
33 + get_tensor_model_parallel_rank,
34 get_tensor_model_parallel_world_size)
35 from vllm.model_executor.layers.activation import SiluAndMul
36 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -46,7 +47,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
47 from vllm.model_executor.model_loader.weight_utils import (
48 default_weight_loader, kv_cache_scales_loader)
49 from vllm.model_executor.sampling_metadata import SamplingMetadata
49 -from vllm.sequence import SamplerOutput
50 +from vllm.sequence import IntermediateTensors, SamplerOutput
51 from vllm.utils import is_hip, print_warning_once
52
53 from .interfaces import SupportsLoRA
@@ -261,12 +262,20 @@ class LlamaModel(nn.Module):
262 config.hidden_size,
263 org_num_embeddings=config.vocab_size,
264 )
264 - self.layers = nn.ModuleList([
265 - LlamaDecoderLayer(config=config,
266 - cache_config=cache_config,
267 - quant_config=quant_config)
268 - for idx in range(config.num_hidden_layers)
269 - ])
265 + self.start_layer, self.end_layer = get_pp_indices(
266 + config.num_hidden_layers,
267 + get_pp_group().rank_in_group,
268 + get_pp_group().world_size)
269 + self.layers = nn.ModuleList(
270 + [nn.Identity() for _ in range(self.start_layer)] + [
271 + LlamaDecoderLayer(config=config,
272 + cache_config=cache_config,
273 + quant_config=quant_config)
274 + for _ in range(self.start_layer, self.end_layer)
275 + ] + [
276 + nn.Identity()
277 + for _ in range(self.end_layer, config.num_hidden_layers)
278 + ])
279 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
280
281 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
@@ -278,22 +287,36 @@ class LlamaModel(nn.Module):
287 positions: torch.Tensor,
288 kv_caches: List[torch.Tensor],
289 attn_metadata: AttentionMetadata,
290 + intermediate_tensors: Optional[IntermediateTensors],
291 inputs_embeds: Optional[torch.Tensor] = None,
282 - ) -> torch.Tensor:
283 - if inputs_embeds is not None:
284 - hidden_states = inputs_embeds
292 + ) -> Union[torch.Tensor, IntermediateTensors]:
293 + if get_pp_group().is_first_rank:
294 + if inputs_embeds is not None:
295 + hidden_states = inputs_embeds
296 + else:
297 + hidden_states = self.get_input_embeddings(input_ids)
298 + residual = None
299 else:
286 - hidden_states = self.get_input_embeddings(input_ids)
287 - residual = None
288 - for i in range(len(self.layers)):
300 + assert intermediate_tensors is not None
301 + hidden_states = intermediate_tensors["hidden_states"]
302 + residual = intermediate_tensors["residual"]
303 +
304 + for i in range(self.start_layer, self.end_layer):
305 layer = self.layers[i]
306 hidden_states, residual = layer(
307 positions,
308 hidden_states,
293 - kv_caches[i],
309 + kv_caches[i - self.start_layer],
310 attn_metadata,
311 residual,
312 )
313 +
314 + if not get_pp_group().is_last_rank:
315 + return IntermediateTensors({
316 + "hidden_states": hidden_states,
317 + "residual": residual
318 + })
319 +
320 hidden_states, _ = self.norm(hidden_states, residual)
321 return hidden_states
322
@@ -372,10 +395,11 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
395 positions: torch.Tensor,
396 kv_caches: List[torch.Tensor],
397 attn_metadata: AttentionMetadata,
375 - ) -> torch.Tensor:
376 - hidden_states = self.model(input_ids, positions, kv_caches,
377 - attn_metadata)
378 - return hidden_states
398 + intermediate_tensors: Optional[IntermediateTensors] = None,
399 + ) -> Union[torch.Tensor, IntermediateTensors]:
400 + model_output = self.model(input_ids, positions, kv_caches,
401 + attn_metadata, intermediate_tensors)
402 + return model_output
403
404 def compute_logits(self, hidden_states: torch.Tensor,
405 sampling_metadata: SamplingMetadata) -> torch.Tensor:
@@ -391,6 +415,20 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
415 next_tokens = self.sampler(logits, sampling_metadata)
416 return next_tokens
417
418 + def make_empty_intermediate_tensors(
419 + self, batch_size: int, dtype: torch.dtype,
420 + device: torch.device) -> IntermediateTensors:
421 + return IntermediateTensors({
422 + "hidden_states":
423 + torch.zeros((batch_size, self.config.hidden_size),
424 + dtype=dtype,
425 + device=device),
426 + "residual":
427 + torch.zeros((batch_size, self.config.hidden_size),
428 + dtype=dtype,
429 + device=device),
430 + })
431 +
432 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
433 stacked_params_mapping = [
434 # (param_name, shard_name, shard_id)
@@ -416,9 +454,12 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
454 # Skip loading extra bias for GPTQ models.
455 if name.endswith(".bias") and name not in params_dict:
456 continue
419 - param = params_dict[name]
420 - weight_loader = param.weight_loader
421 - weight_loader(param, loaded_weight, shard_id)
457 + try:
458 + param = params_dict[name]
459 + weight_loader = param.weight_loader
460 + weight_loader(param, loaded_weight, shard_id)
461 + except KeyError:
462 + pass
463 break
464 else:
465 # Skip loading extra bias for GPTQ models.
@@ -437,10 +478,13 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
478 continue
479 else:
480 name = remapped_kv_scale_name
440 - param = params_dict[name]
441 - weight_loader = getattr(param, "weight_loader",
442 - default_weight_loader)
443 - weight_loader(param, loaded_weight)
481 + try:
482 + param = params_dict[name]
483 + weight_loader = getattr(param, "weight_loader",
484 + default_weight_loader)
485 + weight_loader(param, loaded_weight)
486 + except KeyError:
487 + pass
488
489 # If this function is called, it should always initialize KV cache scale
490 # factors (or else raise an exception). Thus, handled exceptions should
@@ -452,7 +496,8 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
496 quantization_param_path, tp_rank, tp_size,
497 self.config.num_hidden_layers,
498 self.config.__class__.model_type):
455 - layer_self_attn = self.model.layers[layer_idx].self_attn
499 + if not isinstance(self.model.layers[layer_idx], nn.Identity):
500 + layer_self_attn = self.model.layers[layer_idx].self_attn
501
502 if is_hip():
503 # The scaling factor convention we are assuming is
vllm/model_executor/models/llava.py
+3 -1
@@ -18,7 +18,7 @@ from vllm.model_executor.models.clip import CLIPVisionModel
18 from vllm.model_executor.models.llama import LlamaModel
19 from vllm.model_executor.sampling_metadata import SamplingMetadata
20 from vllm.multimodal import MULTIMODAL_REGISTRY
21 -from vllm.sequence import SamplerOutput
21 +from vllm.sequence import IntermediateTensors, SamplerOutput
22
23 from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
24 from .interfaces import SupportsVision
@@ -202,6 +202,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
202 positions: torch.Tensor,
203 kv_caches: List[torch.Tensor],
204 attn_metadata: AttentionMetadata,
205 + intermediate_tensors: Optional[IntermediateTensors] = None,
206 **kwargs: object,
207 ) -> SamplerOutput:
208 """Run forward pass for LLaVA-1.5.
@@ -247,6 +248,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
248 positions,
249 kv_caches,
250 attn_metadata,
251 + None,
252 inputs_embeds=inputs_embeds)
253
254 return hidden_states
vllm/model_executor/models/llava_next.py
+3 -1
@@ -22,7 +22,7 @@ from vllm.model_executor.models.clip import CLIPVisionModel
22 from vllm.model_executor.models.llama import LlamaModel
23 from vllm.model_executor.sampling_metadata import SamplingMetadata
24 from vllm.multimodal import MULTIMODAL_REGISTRY
25 -from vllm.sequence import SamplerOutput
25 +from vllm.sequence import IntermediateTensors, SamplerOutput
26
27 from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
28 get_clip_patch_grid_length)
@@ -376,6 +376,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
376 positions: torch.Tensor,
377 kv_caches: List[torch.Tensor],
378 attn_metadata: AttentionMetadata,
379 + intermediate_tensors: Optional[IntermediateTensors] = None,
380 **kwargs: object,
381 ) -> SamplerOutput:
382 """Run forward pass for LlaVA-NeXT.
@@ -430,6 +431,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
431 positions,
432 kv_caches,
433 attn_metadata,
434 + None,
435 inputs_embeds=inputs_embeds)
436
437 return hidden_states
vllm/model_executor/models/minicpm.py
+2 -1
@@ -50,7 +50,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
50 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
51 from vllm.model_executor.sampling_metadata import SamplingMetadata
52 from vllm.model_executor.utils import set_weight_attrs
53 -from vllm.sequence import SamplerOutput
53 +from vllm.sequence import IntermediateTensors, SamplerOutput
54
55 from .interfaces import SupportsLoRA
56
@@ -462,6 +462,7 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
462 positions: torch.Tensor,
463 kv_caches: List[torch.Tensor],
464 attn_metadata: AttentionMetadata,
465 + intermediate_tensors: Optional[IntermediateTensors] = None,
466 ) -> torch.Tensor:
467 hidden_states = self.model(input_ids, positions, kv_caches,
468 attn_metadata)
vllm/model_executor/models/mixtral.py
+2 -1
@@ -51,7 +51,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
51 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
52 from vllm.model_executor.sampling_metadata import SamplingMetadata
53 from vllm.model_executor.utils import set_weight_attrs
54 -from vllm.sequence import SamplerOutput
54 +from vllm.sequence import IntermediateTensors, SamplerOutput
55 from vllm.utils import print_warning_once
56
57 from .interfaces import SupportsLoRA
@@ -536,6 +536,7 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
536 positions: torch.Tensor,
537 kv_caches: List[torch.Tensor],
538 attn_metadata: AttentionMetadata,
539 + intermediate_tensors: Optional[IntermediateTensors] = None,
540 ) -> torch.Tensor:
541 hidden_states = self.model(input_ids, positions, kv_caches,
542 attn_metadata)
vllm/model_executor/models/mixtral_quant.py
+2 -1
@@ -47,7 +47,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
47 ParallelLMHead, VocabParallelEmbedding)
48 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
49 from vllm.model_executor.sampling_metadata import SamplingMetadata
50 -from vllm.sequence import SamplerOutput
50 +from vllm.sequence import IntermediateTensors, SamplerOutput
51
52
53 class MixtralMLP(nn.Module):
@@ -354,6 +354,7 @@ class MixtralForCausalLM(nn.Module):
354 positions: torch.Tensor,
355 kv_caches: List[torch.Tensor],
356 attn_metadata: AttentionMetadata,
357 + intermediate_tensors: Optional[IntermediateTensors] = None,
358 ) -> torch.Tensor:
359 hidden_states = self.model(input_ids, positions, kv_caches,
360 attn_metadata)
vllm/model_executor/models/mpt.py
+2 -1
@@ -22,7 +22,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
22 VocabParallelEmbedding)
23 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24 from vllm.model_executor.sampling_metadata import SamplingMetadata
25 -from vllm.sequence import SamplerOutput
25 +from vllm.sequence import IntermediateTensors, SamplerOutput
26 from vllm.transformers_utils.configs.mpt import MPTConfig
27
28
@@ -273,6 +273,7 @@ class MPTForCausalLM(nn.Module):
273 positions: torch.Tensor,
274 kv_caches: List[torch.Tensor],
275 attn_metadata: AttentionMetadata,
276 + intermediate_tensors: Optional[IntermediateTensors] = None,
277 ) -> torch.Tensor:
278 hidden_states = self.transformer(input_ids, positions, kv_caches,
279 attn_metadata)
vllm/model_executor/models/olmo.py
+2 -1
@@ -43,7 +43,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
43 ParallelLMHead, VocabParallelEmbedding)
44 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45 from vllm.model_executor.sampling_metadata import SamplingMetadata
46 -from vllm.sequence import SamplerOutput
46 +from vllm.sequence import IntermediateTensors, SamplerOutput
47
48
49 class OlmoAttention(nn.Module):
@@ -301,6 +301,7 @@ class OlmoForCausalLM(nn.Module):
301 positions: torch.Tensor,
302 kv_caches: List[torch.Tensor],
303 attn_metadata: AttentionMetadata,
304 + intermediate_tensors: Optional[IntermediateTensors] = None,
305 ) -> torch.Tensor:
306 hidden_states = self.model(
307 input_ids=input_ids,
vllm/model_executor/models/opt.py
+2 -1
@@ -39,7 +39,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
39 VocabParallelEmbedding)
40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41 from vllm.model_executor.sampling_metadata import SamplingMetadata
42 -from vllm.sequence import SamplerOutput
42 +from vllm.sequence import IntermediateTensors, SamplerOutput
43
44
45 class OPTLearnedPositionalEmbedding(nn.Embedding):
@@ -304,6 +304,7 @@ class OPTForCausalLM(nn.Module):
304 positions: torch.Tensor,
305 kv_caches: List[torch.Tensor],
306 attn_metadata: AttentionMetadata,
307 + intermediate_tensors: Optional[IntermediateTensors] = None,
308 ) -> torch.Tensor:
309 hidden_states = self.model(input_ids, positions, kv_caches,
310 attn_metadata)
vllm/model_executor/models/orion.py
+2 -1
@@ -26,7 +26,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
26 ParallelLMHead, VocabParallelEmbedding)
27 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
28 from vllm.model_executor.sampling_metadata import SamplingMetadata
29 -from vllm.sequence import SamplerOutput
29 +from vllm.sequence import IntermediateTensors, SamplerOutput
30
31
32 class OrionMLP(nn.Module):
@@ -269,6 +269,7 @@ class OrionForCausalLM(nn.Module):
269 positions: torch.Tensor,
270 kv_caches: List[torch.Tensor],
271 attn_metadata: AttentionMetadata,
272 + intermediate_tensors: Optional[IntermediateTensors] = None,
273 ) -> torch.Tensor:
274 hidden_states = self.model(input_ids, positions, kv_caches,
275 attn_metadata)
vllm/model_executor/models/phi.py
+2 -1
@@ -57,7 +57,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
57 ParallelLMHead, VocabParallelEmbedding)
58 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
59 from vllm.model_executor.sampling_metadata import SamplingMetadata
60 -from vllm.sequence import SamplerOutput
60 +from vllm.sequence import IntermediateTensors, SamplerOutput
61
62 from .interfaces import SupportsLoRA
63
@@ -278,6 +278,7 @@ class PhiForCausalLM(nn.Module, SupportsLoRA):
278 positions: torch.Tensor,
279 kv_caches: List[torch.Tensor],
280 attn_metadata: AttentionMetadata,
281 + intermediate_tensors: Optional[IntermediateTensors] = None,
282 ) -> torch.Tensor:
283 hidden_states = self.model(input_ids, positions, kv_caches,
284 attn_metadata)
vllm/model_executor/models/phi3_small.py
+2 -1
@@ -21,7 +21,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
21 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
22 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23 from vllm.model_executor.sampling_metadata import SamplingMetadata
24 -from vllm.sequence import SamplerOutput
24 +from vllm.sequence import IntermediateTensors, SamplerOutput
25
26
27 def load_column_parallel_weight(param: torch.nn.Parameter,
@@ -412,6 +412,7 @@ class Phi3SmallForCausalLM(nn.Module):
412 positions: Optional[torch.LongTensor],
413 kv_caches: List[torch.Tensor],
414 attn_metadata: AttentionMetadata,
415 + intermediate_tensors: Optional[IntermediateTensors] = None,
416 ) -> torch.Tensor:
417 output_hidden_states = self.model(
418 input_ids=input_ids,
vllm/model_executor/models/phi3v.py
+8 -3
@@ -35,7 +35,7 @@ from vllm.model_executor.models.clip import CLIPVisionModel
35 from vllm.model_executor.models.llama import LlamaModel
36 from vllm.model_executor.sampling_metadata import SamplingMetadata
37 from vllm.multimodal import MULTIMODAL_REGISTRY
38 -from vllm.sequence import SamplerOutput
38 +from vllm.sequence import IntermediateTensors, SamplerOutput
39
40 from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
41 from .interfaces import SupportsVision
@@ -381,9 +381,13 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
381
382 return None
383
384 - def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
384 + def forward(self,
385 + input_ids: torch.Tensor,
386 + positions: torch.Tensor,
387 kv_caches: List[torch.Tensor],
386 - attn_metadata: AttentionMetadata, **kwargs: object):
388 + attn_metadata: AttentionMetadata,
389 + intermediate_tensors: Optional[IntermediateTensors] = None,
390 + **kwargs: object):
391 image_input = self._parse_and_validate_image_input(**kwargs)
392
393 if image_input is not None:
@@ -398,6 +402,7 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
402 positions,
403 kv_caches,
404 attn_metadata,
405 + intermediate_tensors,
406 inputs_embeds=inputs_embeds)
407
408 return hidden_states
vllm/model_executor/models/qwen.py
+2 -1
@@ -27,7 +27,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
27 ParallelLMHead, VocabParallelEmbedding)
28 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
29 from vllm.model_executor.sampling_metadata import SamplingMetadata
30 -from vllm.sequence import SamplerOutput
30 +from vllm.sequence import IntermediateTensors, SamplerOutput
31 from vllm.utils import print_warning_once
32
33
@@ -245,6 +245,7 @@ class QWenLMHeadModel(nn.Module):
245 positions: torch.Tensor,
246 kv_caches: List[torch.Tensor],
247 attn_metadata: AttentionMetadata,
248 + intermediate_tensors: Optional[IntermediateTensors] = None,
249 ) -> torch.Tensor:
250 hidden_states = self.transformer(input_ids, positions, kv_caches,
251 attn_metadata)
vllm/model_executor/models/qwen2.py
+2 -1
@@ -45,7 +45,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
45 ParallelLMHead, VocabParallelEmbedding)
46 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
47 from vllm.model_executor.sampling_metadata import SamplingMetadata
48 -from vllm.sequence import SamplerOutput
48 +from vllm.sequence import IntermediateTensors, SamplerOutput
49 from vllm.utils import print_warning_once
50
51 from .interfaces import SupportsLoRA
@@ -331,6 +331,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
331 positions: torch.Tensor,
332 kv_caches: List[torch.Tensor],
333 attn_metadata: AttentionMetadata,
334 + intermediate_tensors: Optional[IntermediateTensors] = None,
335 ) -> torch.Tensor:
336 hidden_states = self.model(input_ids, positions, kv_caches,
337 attn_metadata)
vllm/model_executor/models/qwen2_moe.py
+2 -1
@@ -50,7 +50,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
50 ParallelLMHead, VocabParallelEmbedding)
51 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
52 from vllm.model_executor.sampling_metadata import SamplingMetadata
53 -from vllm.sequence import SamplerOutput
53 +from vllm.sequence import IntermediateTensors, SamplerOutput
54
55
56 class Qwen2MoeMLP(nn.Module):
@@ -397,6 +397,7 @@ class Qwen2MoeForCausalLM(nn.Module):
397 positions: torch.Tensor,
398 kv_caches: List[torch.Tensor],
399 attn_metadata: AttentionMetadata,
400 + intermediate_tensors: Optional[IntermediateTensors] = None,
401 ) -> torch.Tensor:
402 hidden_states = self.model(input_ids, positions, kv_caches,
403 attn_metadata)
vllm/model_executor/models/stablelm.py
+2 -1
@@ -41,7 +41,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
41 ParallelLMHead, VocabParallelEmbedding)
42 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
43 from vllm.model_executor.sampling_metadata import SamplingMetadata
44 -from vllm.sequence import SamplerOutput
44 +from vllm.sequence import IntermediateTensors, SamplerOutput
45
46
47 class StablelmMLP(nn.Module):
@@ -250,6 +250,7 @@ class StablelmForCausalLM(nn.Module):
250 positions: torch.Tensor,
251 kv_caches: List[torch.Tensor],
252 attn_metadata: AttentionMetadata,
253 + intermediate_tensors: Optional[IntermediateTensors] = None,
254 ) -> torch.Tensor:
255 hidden_states = self.model(input_ids, positions, kv_caches,
256 attn_metadata)
vllm/model_executor/models/starcoder2.py
+2 -1
@@ -40,7 +40,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
40 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
41 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
42 from vllm.model_executor.sampling_metadata import SamplingMetadata
43 -from vllm.sequence import SamplerOutput
43 +from vllm.sequence import IntermediateTensors, SamplerOutput
44
45
46 class Starcoder2Attention(nn.Module):
@@ -262,6 +262,7 @@ class Starcoder2ForCausalLM(nn.Module):
262 positions: torch.Tensor,
263 kv_caches: List[torch.Tensor],
264 attn_metadata: AttentionMetadata,
265 + intermediate_tensors: Optional[IntermediateTensors] = None,
266 ) -> torch.Tensor:
267 hidden_states = self.model(input_ids, positions, kv_caches,
268 attn_metadata)
vllm/model_executor/models/xverse.py
+2 -1
@@ -43,7 +43,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
43 ParallelLMHead, VocabParallelEmbedding)
44 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45 from vllm.model_executor.sampling_metadata import SamplingMetadata
46 -from vllm.sequence import SamplerOutput
46 +from vllm.sequence import IntermediateTensors, SamplerOutput
47
48 from .interfaces import SupportsLoRA
49
@@ -320,6 +320,7 @@ class XverseForCausalLM(nn.Module, SupportsLoRA):
320 positions: torch.Tensor,
321 kv_caches: List[torch.Tensor],
322 attn_metadata: AttentionMetadata,
323 + intermediate_tensors: Optional[IntermediateTensors] = None,
324 ) -> torch.Tensor:
325 hidden_states = self.model(input_ids, positions, kv_caches,
326 attn_metadata)
vllm/sequence.py
+31
@@ -770,6 +770,34 @@ class EmbeddingSequenceGroupOutput(SequenceGroupOutput):
770 return self.embeddings == other.embeddings
771
772
773 +@dataclass
774 +class IntermediateTensors:
775 + """For all pipeline stages except the last, we need to return the hidden
776 + states and residuals to be sent to the next stage. This data structure
777 + contains the hidden states and residuals for a request.
778 + """
779 +
780 + tensors: Dict[str, torch.Tensor]
781 +
782 + def __getitem__(self, key: Union[str, slice]):
783 + if isinstance(key, str):
784 + return self.tensors[key]
785 + elif isinstance(key, slice):
786 + return self.__class__({k: v[key] for k, v in self.tensors.items()})
787 +
788 + def __setitem__(self, key: str, value):
789 + self.tensors[key] = value
790 +
791 + def __len__(self):
792 + return len(self.tensors)
793 +
794 + def __eq__(self, other: object):
795 + return isinstance(other, self.__class__) and self
796 +
797 + def __repr__(self) -> str:
798 + return f"IntermediateTensors(tensors={self.tensors})"
799 +
800 +
801 @dataclass
802 class SamplerOutput:
803 """For each sequence group, we generate a list of SequenceOutput object,
@@ -896,6 +924,8 @@ class ExecuteModelRequest:
924 blocks_to_swap_out: List[Tuple[int, int]] = field(default_factory=list)
925 # Blocks to copy. Source to dest block.
926 blocks_to_copy: List[Tuple[int, int]] = field(default_factory=list)
927 + # Virtual engine ID for pipeline parallel.
928 + virtual_engine: int = 0
929 # The number of slots for lookahead decoding.
930 num_lookahead_slots: int = 0
931 # The number of requests in the running queue.
@@ -914,6 +944,7 @@ class ExecuteModelRequest:
944 blocks_to_swap_in=self.blocks_to_swap_in.copy(),
945 blocks_to_swap_out=self.blocks_to_swap_out.copy(),
946 blocks_to_copy=self.blocks_to_copy.copy(),
947 + virtual_engine=self.virtual_engine,
948 num_lookahead_slots=self.num_lookahead_slots,
949 running_queue_size=self.running_queue_size,
950 previous_hidden_states=self.previous_hidden_states,
vllm/spec_decode/draft_model_runner.py
+10 -5
@@ -6,7 +6,8 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6 ModelConfig, ParallelConfig, SchedulerConfig,
7 VisionLanguageConfig)
8 from vllm.logger import init_logger
9 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
9 +from vllm.sequence import (IntermediateTensors, SamplerOutput,
10 + SequenceGroupMetadata)
11 from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
12 ModelRunner)
13
@@ -74,9 +75,9 @@ class TP1DraftModelRunner(ModelRunner):
75 List[SequenceGroupMetadata]] = None
76
77 def prepare_model_input(
77 - self,
78 - seq_group_metadata_list: List[SequenceGroupMetadata],
79 - ) -> ModelInputForGPUWithSamplingMetadata:
78 + self,
79 + seq_group_metadata_list: List[SequenceGroupMetadata],
80 + virtual_engine: int = 0) -> ModelInputForGPUWithSamplingMetadata:
81 """A temporary solution that caches the seq_group_metadata_list
82 for multi-step execution.
83 TODO: In-place update model_input and remove this function.
@@ -115,6 +116,7 @@ class TP1DraftModelRunner(ModelRunner):
116 self,
117 model_input: ModelInputForGPUWithSamplingMetadata,
118 kv_caches: List[torch.Tensor],
119 + intermediate_tensors: Optional[IntermediateTensors] = None,
120 num_steps: int = 1,
121 ) -> Optional[List[SamplerOutput]]:
122 # Since we do not broadcast data inside execute_model anymore,
@@ -130,6 +132,7 @@ class TP1DraftModelRunner(ModelRunner):
132 self.set_active_loras(model_input.lora_requests,
133 model_input.lora_mapping)
134
135 + virtual_engine = model_input.virtual_engine
136 outputs: List[SamplerOutput] = []
137 for step in range(num_steps):
138 # Currently cuda graph is only supported by the decode phase.
@@ -139,7 +142,8 @@ class TP1DraftModelRunner(ModelRunner):
142 if prefill_meta is None and decode_meta.use_cuda_graph:
143 assert model_input.input_tokens is not None
144 graph_batch_size = model_input.input_tokens.shape[0]
142 - model_executable = self.graph_runners[graph_batch_size]
145 + model_executable = (
146 + self.graph_runners[virtual_engine][graph_batch_size])
147 else:
148 model_executable = self.model
149
@@ -149,6 +153,7 @@ class TP1DraftModelRunner(ModelRunner):
153 positions=model_input.input_positions,
154 kv_caches=kv_caches,
155 attn_metadata=model_input.attn_metadata,
156 + intermediate_tensors=intermediate_tensors,
157 **multi_modal_kwargs,
158 )
159
vllm/worker/cache_engine.py
+4
@@ -38,7 +38,11 @@ class CacheEngine:
38
39 self.block_size = cache_config.block_size
40 self.num_gpu_blocks = cache_config.num_gpu_blocks
41 + if self.num_gpu_blocks:
42 + self.num_gpu_blocks //= parallel_config.pipeline_parallel_size
43 self.num_cpu_blocks = cache_config.num_cpu_blocks
44 + if self.num_cpu_blocks:
45 + self.num_cpu_blocks //= parallel_config.pipeline_parallel_size
46
47 if cache_config.cache_dtype == "auto":
48 self.dtype = model_config.dtype
vllm/worker/cpu_model_runner.py
+4 -1
@@ -13,7 +13,8 @@ from vllm.logger import init_logger
13 from vllm.model_executor import SamplingMetadata
14 from vllm.model_executor.model_loader import get_model
15 from vllm.multimodal import MULTIMODAL_REGISTRY
16 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
16 +from vllm.sequence import (IntermediateTensors, SamplerOutput,
17 + SequenceGroupMetadata)
18 from vllm.utils import make_tensor_with_pad
19 from vllm.worker.model_runner_base import (
20 ModelRunnerBase, ModelRunnerInputBase,
@@ -315,6 +316,7 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
316 def prepare_model_input(
317 self,
318 seq_group_metadata_list: List[SequenceGroupMetadata],
319 + virtual_engine: int = 0,
320 ) -> CPUModelInput:
321 multi_modal_kwargs = None
322 # NOTE: We assume that all sequences in the group are all prompts or
@@ -351,6 +353,7 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
353 self,
354 model_input: CPUModelInput,
355 kv_caches: List[torch.Tensor],
356 + intermediate_tensors: Optional[IntermediateTensors] = None,
357 num_steps: int = 1,
358 ) -> Optional[List[SamplerOutput]]:
359 if num_steps > 1:
vllm/worker/cpu_worker.py
+24 -14
@@ -167,8 +167,8 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
167 is_driver_worker=is_driver_worker)
168 # Uninitialized cache engine. Will be initialized by
169 # initialize_cache.
170 - self.cache_engine: CPUCacheEngine
171 - self.cpu_cache: List[torch.Tensor]
170 + self.cache_engine: List[CPUCacheEngine]
171 + self.cpu_cache: List[List[torch.Tensor]]
172
173 def init_device(self) -> None:
174 self.init_distributed_environment()
@@ -242,25 +242,32 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
242 "initializing the engine.")
243
244 def _init_cache_engine(self) -> None:
245 - self.cache_engine = CPUCacheEngine(self.cache_config,
246 - self.model_config,
247 - self.parallel_config,
248 - self.device_config)
249 - self.cpu_cache = self.cache_engine.cpu_cache
250 - self.model_runner.block_size = self.cache_engine.block_size
251 -
252 - assert self.cpu_cache is not None
245 + self.cache_engine = [
246 + CPUCacheEngine(self.cache_config, self.model_config,
247 + self.parallel_config, self.device_config)
248 + for _ in range(self.parallel_config.pipeline_parallel_size)
249 + ]
250 + self.cpu_cache = [
251 + self.cache_engine[ve].cpu_cache
252 + for ve in range(self.parallel_config.pipeline_parallel_size)
253 + ]
254 + self.model_runner.block_size = self.cache_engine[0].block_size
255 +
256 + assert all(
257 + self.cpu_cache[ve] is not None
258 + for ve in range(self.parallel_config.pipeline_parallel_size))
259
260 # Populate the cache to warmup the memory
255 - for layer_cache in self.cpu_cache:
256 - layer_cache.fill_(0)
261 + for ve in range(self.parallel_config.pipeline_parallel_size):
262 + for layer_cache in self.cpu_cache[ve]:
263 + layer_cache.fill_(0)
264
265 @property
266 def do_metadata_broadcast(self) -> bool:
267 return self.parallel_config.tensor_parallel_size > 1
268
269 @property
263 - def kv_cache(self) -> Optional[List[torch.Tensor]]:
270 + def kv_cache(self) -> Optional[List[List[torch.Tensor]]]:
271 return self.cpu_cache
272
273 def execute_worker(
@@ -269,12 +276,14 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
276 ) -> None:
277 if (worker_input.blocks_to_copy is not None
278 and worker_input.blocks_to_copy.numel() > 0):
272 - self.cache_engine.copy(worker_input.blocks_to_copy)
279 + self.cache_engine[worker_input.virtual_engine].copy(
280 + worker_input.blocks_to_copy)
281
282 @torch.inference_mode()
283 def prepare_worker_input(
284 self, execute_model_req: ExecuteModelRequest) -> WorkerInput:
285 assert execute_model_req is not None
286 + virtual_engine = execute_model_req.virtual_engine
287 num_seq_groups: int = len(execute_model_req.seq_group_metadata_list)
288 blocks_to_copy = execute_model_req.blocks_to_copy
289 blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
@@ -285,6 +294,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
294 return WorkerInput(
295 num_seq_groups=num_seq_groups,
296 blocks_to_copy=blocks_to_copy,
297 + virtual_engine=virtual_engine,
298 )
299
300 def init_distributed_environment(self) -> None:
vllm/worker/embedding_model_runner.py
+7 -2
@@ -9,7 +9,8 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
9 from vllm.logger import init_logger
10 from vllm.model_executor.pooling_metadata import PoolingMetadata
11 from vllm.pooling_params import PoolingParams
12 -from vllm.sequence import PoolerOutput, SequenceData, SequenceGroupMetadata
12 +from vllm.sequence import (IntermediateTensors, PoolerOutput, SequenceData,
13 + SequenceGroupMetadata)
14 from vllm.worker.model_runner import GPUModelRunnerBase, ModelInputForGPU
15
16 logger = init_logger(__name__)
@@ -57,6 +58,7 @@ class EmbeddingModelRunner(
58 self,
59 model_input: ModelInputForGPUWithPoolingMetadata,
60 kv_caches: List[torch.Tensor],
61 + intermediate_tensors: Optional[IntermediateTensors] = None,
62 num_steps: int = 1,
63 ) -> Optional[List[PoolerOutput]]:
64 if num_steps > 1:
@@ -73,10 +75,12 @@ class EmbeddingModelRunner(
75 assert model_input.attn_metadata is not None
76 prefill_meta = model_input.attn_metadata.prefill_metadata
77 decode_meta = model_input.attn_metadata.decode_metadata
78 + virtual_engine = model_input.virtual_engine
79 if prefill_meta is None and decode_meta.use_cuda_graph:
80 assert model_input.input_tokens is not None
81 graph_batch_size = model_input.input_tokens.shape[0]
79 - model_executable = self.graph_runners[graph_batch_size]
82 + model_executable = self.graph_runners[virtual_engine][
83 + graph_batch_size]
84 else:
85 model_executable = self.model
86
@@ -115,6 +119,7 @@ class EmbeddingModelRunner(
119 def prepare_model_input(
120 self,
121 seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
122 + virtual_engine: int = 0,
123 ) -> ModelInputForGPUWithPoolingMetadata:
124 assert seq_group_metadata_list is not None
125 model_input = self._prepare_model_input_tensors(
vllm/worker/model_runner.py
+201 -125
@@ -8,6 +8,7 @@ from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Type,
8
9 import numpy as np
10 import torch
11 +import torch.distributed
12 import torch.nn as nn
13
14 try:
@@ -25,6 +26,7 @@ from vllm.attention import AttentionMetadata, get_attn_backend
26 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
27 ModelConfig, ParallelConfig, SchedulerConfig,
28 VisionLanguageConfig)
29 +from vllm.distributed import get_pp_group
30 from vllm.distributed.parallel_state import graph_capture
31 from vllm.inputs import INPUT_REGISTRY
32 from vllm.logger import init_logger
@@ -37,7 +39,8 @@ from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
39 from vllm.model_executor.models.interfaces import supports_lora
40 from vllm.multimodal import MULTIMODAL_REGISTRY
41 from vllm.sampling_params import SamplingParams
40 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
42 +from vllm.sequence import (IntermediateTensors, SamplerOutput,
43 + SequenceGroupMetadata)
44 from vllm.utils import (CudaMemoryProfiler, get_kv_cache_torch_dtype, is_hip,
45 is_pin_memory_available, make_tensor_with_pad)
46 from vllm.worker.model_runner_base import (
@@ -81,6 +84,7 @@ class ModelInputForGPU(ModelRunnerInputBase):
84 lora_requests: Optional[Set[LoRARequest]] = None
85 attn_metadata: Optional["AttentionMetadata"] = None
86 multi_modal_kwargs: Optional[Dict[str, torch.Tensor]] = None
87 + virtual_engine: int = 0
88
89 def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
90 tensor_dict = {
@@ -89,6 +93,7 @@ class ModelInputForGPU(ModelRunnerInputBase):
93 "lora_requests": self.lora_requests,
94 "lora_mapping": self.lora_mapping,
95 "multi_modal_kwargs": self.multi_modal_kwargs,
96 + "virtual_engine": self.virtual_engine,
97 }
98 _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
99 return tensor_dict
@@ -122,6 +127,7 @@ class ModelInputForGPUWithSamplingMetadata(ModelInputForGPU):
127 "lora_requests": self.lora_requests,
128 "lora_mapping": self.lora_mapping,
129 "multi_modal_kwargs": self.multi_modal_kwargs,
130 + "virtual_engine": self.virtual_engine,
131 }
132 _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
133 _add_sampling_metadata_broadcastable_dict(tensor_dict,
@@ -179,7 +185,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
185 self.sliding_window = model_config.get_sliding_window()
186 self.block_size = cache_config.block_size
187 self.max_seq_len_to_capture = self.model_config.max_seq_len_to_capture
182 - self.graph_runners: Dict[int, CUDAGraphRunner] = {}
188 +
189 + self.graph_runners: List[Dict[int, CUDAGraphRunner]] = [
190 + {} for _ in range(self.parallel_config.pipeline_parallel_size)
191 + ]
192 self.graph_memory_pool: Optional[Tuple[
193 int, int]] = None # Set during graph capture.
194 # When using CUDA graph, the input block tables must be padded to
@@ -787,9 +796,11 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
796 max_num_seqs = min(
797 max_num_seqs,
798 int(max_num_batched_tokens / vlm_config.image_feature_size))
799 + batch_size = 0
800 for group_id in range(max_num_seqs):
801 seq_len = (max_num_batched_tokens // max_num_seqs +
802 (group_id < max_num_batched_tokens % max_num_seqs))
803 + batch_size += seq_len
804
805 seq_data, dummy_multi_modal_data = INPUT_REGISTRY \
806 .dummy_data_for_profiling(model_config, seq_len)
@@ -811,7 +822,13 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
822 num_layers = self.model_config.get_num_layers(self.parallel_config)
823 kv_caches = [None] * num_layers
824 model_input = self.prepare_model_input(seqs)
814 - self.execute_model(model_input, kv_caches)
825 + intermediate_tensors = None
826 + if not get_pp_group().is_first_rank:
827 + intermediate_tensors = self.model.make_empty_intermediate_tensors(
828 + batch_size=batch_size,
829 + dtype=self.model_config.dtype,
830 + device=self.device)
831 + self.execute_model(model_input, kv_caches, intermediate_tensors)
832 torch.cuda.synchronize()
833 return
834
@@ -847,7 +864,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
864 return self.lora_manager.list_loras()
865
866 @torch.inference_mode()
850 - def capture_model(self, kv_caches: List[torch.Tensor]) -> None:
867 + def capture_model(self, kv_caches: List[List[torch.Tensor]]) -> None:
868 """Cuda graph capture a model.
869
870 Note that CUDA graph's performance gain is negligible if number
@@ -880,10 +897,18 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
897 slot_mapping.fill_(_PAD_SLOT_ID)
898 seq_lens = torch.ones(max_batch_size, dtype=torch.int32).cuda()
899 block_tables = torch.from_numpy(self.graph_block_tables).cuda()
900 + intermediate_inputs = None
901 + if not get_pp_group().is_first_rank:
902 + intermediate_inputs = self.model.make_empty_intermediate_tensors(
903 + batch_size=max_batch_size,
904 + dtype=self.model_config.dtype,
905 + device=self.device)
906
907 # Prepare buffer for outputs. These will be reused for all batch sizes.
908 # It will be filled after the first graph capture.
886 - hidden_states: Optional[torch.Tensor] = None
909 + hidden_or_intermediate_states: List[Optional[torch.Tensor]] = [
910 + None
911 + ] * self.parallel_config.pipeline_parallel_size
912
913 graph_batch_size = _get_graph_batch_size(
914 self.scheduler_config.max_num_seqs)
@@ -912,109 +937,120 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
937 with graph_capture() as graph_capture_context:
938 # NOTE: Capturing the largest batch size first may help reduce the
939 # memory usage of CUDA graph.
915 - for batch_size in reversed(batch_size_capture_list):
916 - if self.attn_backend.get_name() == "flashinfer":
917 - indptr_buffer = indptr_buffer[:batch_size + 1]
918 - last_page_len_buffer = last_page_len_buffer[:batch_size]
919 -
920 - num_qo_heads = self.model_config.get_num_attention_heads(
921 - self.parallel_config)
922 - num_kv_heads = self.model_config.get_num_kv_heads(
923 - self.parallel_config)
924 - if num_qo_heads // num_kv_heads >= 4:
925 - use_tensor_cores = True
940 + for virtual_engine in range(
941 + self.parallel_config.pipeline_parallel_size):
942 + for batch_size in reversed(batch_size_capture_list):
943 + if self.attn_backend.get_name() == "flashinfer":
944 + indptr_buffer = indptr_buffer[:batch_size + 1]
945 + last_page_len_buffer = last_page_len_buffer[:
946 + batch_size]
947 +
948 + num_qo_heads = (
949 + self.model_config.get_num_attention_heads(
950 + self.parallel_config))
951 + num_kv_heads = self.model_config.get_num_kv_heads(
952 + self.parallel_config)
953 + if num_qo_heads // num_kv_heads >= 4:
954 + use_tensor_cores = True
955 + else:
956 + use_tensor_cores = False
957 + decode_wrapper = \
958 + CUDAGraphBatchDecodeWithPagedKVCacheWrapper(
959 + decode_workspace_buffer, indptr_buffer,
960 + indices_buffer, last_page_len_buffer, "NHD",
961 + use_tensor_cores)
962 + kv_cache_dtype = get_kv_cache_torch_dtype(
963 + self.kv_cache_dtype, self.model_config.dtype)
964 +
965 + paged_kv_indptr_tensor_host = torch.arange(
966 + 0, batch_size + 1, dtype=torch.int32)
967 + paged_kv_indices_tensor_host = torch.arange(
968 + 0, batch_size, dtype=torch.int32)
969 + paged_kv_last_page_len_tensor_host = torch.full(
970 + (batch_size, ), self.block_size, dtype=torch.int32)
971 + query_start_loc_host = torch.arange(0,
972 + batch_size + 1,
973 + dtype=torch.int32)
974 +
975 + attn_metadata = self.attn_backend.make_metadata(
976 + num_prefills=0,
977 + slot_mapping=slot_mapping[:batch_size],
978 + num_prefill_tokens=0,
979 + num_decode_tokens=batch_size,
980 + max_prefill_seq_len=0,
981 + block_tables=block_tables,
982 + paged_kv_indptr=paged_kv_indptr_tensor_host,
983 + paged_kv_indices=paged_kv_indices_tensor_host,
984 + paged_kv_last_page_len=
985 + paged_kv_last_page_len_tensor_host,
986 + num_qo_heads=num_qo_heads,
987 + num_kv_heads=num_kv_heads,
988 + head_dim=self.model_config.get_head_size(),
989 + page_size=self.block_size,
990 + seq_start_loc=None,
991 + query_start_loc=query_start_loc_host,
992 + device=self.device,
993 + data_type=kv_cache_dtype,
994 + use_cuda_graph=True,
995 + decode_wrapper=decode_wrapper,
996 + prefill_wrapper=None)
997 + attn_metadata.begin_forward()
998 else:
927 - use_tensor_cores = False
928 - decode_wrapper = \
929 - CUDAGraphBatchDecodeWithPagedKVCacheWrapper(
930 - decode_workspace_buffer, indptr_buffer, indices_buffer,
931 - last_page_len_buffer, "NHD", use_tensor_cores)
932 - kv_cache_dtype = get_kv_cache_torch_dtype(
933 - self.kv_cache_dtype, self.model_config.dtype)
934 -
935 - paged_kv_indptr_tensor_host = torch.arange(
936 - 0, batch_size + 1, dtype=torch.int32)
937 - paged_kv_indices_tensor_host = torch.arange(
938 - 0, batch_size, dtype=torch.int32)
939 - paged_kv_last_page_len_tensor_host = torch.full(
940 - (batch_size, ), self.block_size, dtype=torch.int32)
941 - query_start_loc_host = torch.arange(0,
942 - batch_size + 1,
943 - dtype=torch.int32)
944 -
945 - attn_metadata = self.attn_backend.make_metadata(
946 - num_prefills=0,
947 - slot_mapping=slot_mapping[:batch_size],
948 - num_prefill_tokens=0,
949 - num_decode_tokens=batch_size,
950 - max_prefill_seq_len=0,
951 - block_tables=block_tables,
952 - paged_kv_indptr=paged_kv_indptr_tensor_host,
953 - paged_kv_indices=paged_kv_indices_tensor_host,
954 - paged_kv_last_page_len=
955 - paged_kv_last_page_len_tensor_host,
956 - num_qo_heads=num_qo_heads,
957 - num_kv_heads=num_kv_heads,
958 - head_dim=self.model_config.get_head_size(),
959 - page_size=self.block_size,
960 - seq_start_loc=None,
961 - query_start_loc=query_start_loc_host,
962 - device=self.device,
963 - data_type=kv_cache_dtype,
964 - use_cuda_graph=True,
965 - decode_wrapper=decode_wrapper,
966 - prefill_wrapper=None)
967 - attn_metadata.begin_forward()
968 - else:
969 - attn_metadata = self.attn_backend.make_metadata(
970 - num_prefills=0,
971 - num_prefill_tokens=0,
972 - num_decode_tokens=batch_size,
973 - slot_mapping=slot_mapping[:batch_size],
974 - seq_lens=None,
975 - seq_lens_tensor=seq_lens[:batch_size],
976 - max_query_len=None,
977 - max_prefill_seq_len=0,
978 - max_decode_seq_len=self.max_seq_len_to_capture,
979 - query_start_loc=None,
980 - seq_start_loc=None,
981 - context_lens_tensor=None,
982 - block_tables=block_tables[:batch_size],
983 - use_cuda_graph=True,
999 + attn_metadata = self.attn_backend.make_metadata(
1000 + num_prefills=0,
1001 + num_prefill_tokens=0,
1002 + num_decode_tokens=batch_size,
1003 + slot_mapping=slot_mapping[:batch_size],
1004 + seq_lens=None,
1005 + seq_lens_tensor=seq_lens[:batch_size],
1006 + max_query_len=None,
1007 + max_prefill_seq_len=0,
1008 + max_decode_seq_len=self.max_seq_len_to_capture,
1009 + query_start_loc=None,
1010 + seq_start_loc=None,
1011 + context_lens_tensor=None,
1012 + block_tables=block_tables[:batch_size],
1013 + use_cuda_graph=True,
1014 + )
1015 +
1016 + if self.lora_config:
1017 + lora_mapping = LoRAMapping(
1018 + [0] * batch_size,
1019 + [0] * batch_size,
1020 + )
1021 + self.set_active_loras(set(), lora_mapping)
1022 +
1023 + graph_runner = CUDAGraphRunner(
1024 + self.model, self.attn_backend.get_name())
1025 +
1026 + if self.attn_backend.get_name() == "flashinfer":
1027 + graph_runner.flashinfer_indptr_buffer = indptr_buffer
1028 + graph_runner.flashinfer_indices_buffer = indices_buffer
1029 + graph_runner.flashinfer_last_page_len_buffer = \
1030 + last_page_len_buffer
1031 + graph_runner.flashinfer_decode_workspace_buffer = \
1032 + decode_workspace_buffer
1033 + graph_runner.flashinfer_decode_wrapper = \
1034 + decode_wrapper
1035 +
1036 + graph_runner.capture(
1037 + input_tokens[:batch_size],
1038 + input_positions[:batch_size],
1039 + hidden_or_intermediate_states[
1040 + virtual_engine] # type: ignore
1041 + [:batch_size]
1042 + if hidden_or_intermediate_states[virtual_engine]
1043 + is not None else None,
1044 + intermediate_inputs[:batch_size]
1045 + if intermediate_inputs is not None else None,
1046 + kv_caches[virtual_engine],
1047 + attn_metadata,
1048 + memory_pool=self.graph_memory_pool,
1049 + stream=graph_capture_context.stream,
1050 )
985 -
986 - if self.lora_config:
987 - lora_mapping = LoRAMapping(
988 - [0] * batch_size,
989 - [0] * batch_size,
990 - )
991 - self.set_active_loras(set(), lora_mapping)
992 -
993 - graph_runner = CUDAGraphRunner(self.model,
994 - self.attn_backend.get_name())
995 -
996 - if self.attn_backend.get_name() == "flashinfer":
997 - graph_runner.flashinfer_indptr_buffer = indptr_buffer
998 - graph_runner.flashinfer_indices_buffer = indices_buffer
999 - graph_runner.flashinfer_last_page_len_buffer = \
1000 - last_page_len_buffer
1001 - graph_runner.flashinfer_decode_workspace_buffer = \
1002 - decode_workspace_buffer
1003 - graph_runner.flashinfer_decode_wrapper = \
1004 - decode_wrapper
1005 -
1006 - graph_runner.capture(
1007 - input_tokens[:batch_size],
1008 - input_positions[:batch_size],
1009 - hidden_states[:batch_size]
1010 - if hidden_states is not None else None,
1011 - kv_caches,
1012 - attn_metadata,
1013 - memory_pool=self.graph_memory_pool,
1014 - stream=graph_capture_context.stream,
1015 - )
1016 - self.graph_memory_pool = graph_runner.graph.pool()
1017 - self.graph_runners[batch_size] = graph_runner
1051 + self.graph_memory_pool = graph_runner.graph.pool()
1052 + self.graph_runners[virtual_engine][batch_size] = (
1053 + graph_runner)
1054
1055 end_time = time.perf_counter()
1056 elapsed_time = end_time - start_time
@@ -1047,6 +1083,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1083 def prepare_model_input(
1084 self,
1085 seq_group_metadata_list: List[SequenceGroupMetadata],
1086 + virtual_engine: int = 0,
1087 ) -> ModelInputForGPUWithSamplingMetadata:
1088 """Prepare the model input based on a given sequence group, including
1089 metadata for the sampling step.
@@ -1072,15 +1109,17 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1109 if seq_group_metadata_list else None)
1110 return dataclasses.replace(model_input,
1111 sampling_metadata=sampling_metadata,
1075 - is_prompt=is_prompt)
1112 + is_prompt=is_prompt,
1113 + virtual_engine=virtual_engine)
1114
1115 @torch.inference_mode()
1116 def execute_model(
1117 self,
1118 model_input: ModelInputForGPUWithSamplingMetadata,
1119 kv_caches: List[torch.Tensor],
1120 + intermediate_tensors: Optional[IntermediateTensors] = None,
1121 num_steps: int = 1,
1083 - ) -> Optional[List[SamplerOutput]]:
1122 + ) -> Optional[Union[List[SamplerOutput], IntermediateTensors]]:
1123 if num_steps > 1:
1124 raise ValueError("num_steps > 1 is not supported in ModelRunner")
1125
@@ -1124,27 +1163,34 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1163 assert model_input.attn_metadata is not None
1164 prefill_meta = model_input.attn_metadata.prefill_metadata
1165 decode_meta = model_input.attn_metadata.decode_metadata
1166 + # TODO(andoorve): We can remove this once all
1167 + # virtual engines share the same kv cache.
1168 + virtual_engine = model_input.virtual_engine
1169 if prefill_meta is None and decode_meta.use_cuda_graph:
1170 assert model_input.input_tokens is not None
1171 graph_batch_size = model_input.input_tokens.shape[0]
1130 - model_executable = self.graph_runners[graph_batch_size]
1172 + model_executable = self.graph_runners[virtual_engine][
1173 + graph_batch_size]
1174 else:
1175 model_executable = self.model
1176
1177 multi_modal_kwargs = model_input.multi_modal_kwargs or {}
1135 - hidden_states = model_executable(
1178 + hidden_or_intermediate_states = model_executable(
1179 input_ids=model_input.input_tokens,
1180 positions=model_input.input_positions,
1181 kv_caches=kv_caches,
1182 attn_metadata=model_input.attn_metadata,
1183 + intermediate_tensors=intermediate_tensors,
1184 **multi_modal_kwargs,
1185 )
1186
1143 - # Compute the logits.
1144 - logits = self.model.compute_logits(hidden_states,
1187 + # Compute the logits in the last pipeline stage.
1188 + if not get_pp_group().is_last_rank:
1189 + return hidden_or_intermediate_states
1190 +
1191 + logits = self.model.compute_logits(hidden_or_intermediate_states,
1192 model_input.sampling_metadata)
1193
1147 - # Only perform sampling in the driver worker.
1194 if not self.is_driver_worker:
1195 return []
1196
@@ -1159,9 +1205,12 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1205 assert model_input.sampling_metadata is not None
1206 indices = model_input.sampling_metadata.selected_token_indices
1207 if model_input.is_prompt:
1162 - hidden_states = hidden_states.index_select(0, indices)
1208 + hidden_states = hidden_or_intermediate_states.index_select(
1209 + 0, indices)
1210 elif decode_meta.use_cuda_graph:
1164 - hidden_states = hidden_states[:len(indices)]
1211 + hidden_states = hidden_or_intermediate_states[:len(indices)]
1212 + else:
1213 + hidden_states = hidden_or_intermediate_states
1214
1215 output.hidden_states = hidden_states
1216
@@ -1195,13 +1244,15 @@ class CUDAGraphRunner:
1244 self,
1245 input_ids: torch.Tensor,
1246 positions: torch.Tensor,
1198 - hidden_states: Optional[torch.Tensor],
1247 + hidden_or_intermediate_states: Optional[Union[IntermediateTensors,
1248 + torch.Tensor]],
1249 + intermediate_inputs: Optional[IntermediateTensors],
1250 kv_caches: List[torch.Tensor],
1251 attn_metadata: AttentionMetadata,
1252 memory_pool: Optional[Tuple[int, int]],
1253 stream: torch.cuda.Stream,
1254 **kwargs,
1204 - ) -> torch.Tensor:
1255 + ) -> Union[torch.Tensor, IntermediateTensors]:
1256 assert self._graph is None
1257 # Run the model a few times without capturing the graph.
1258 # This is to make sure that the captured graph does not include the
@@ -1213,6 +1264,7 @@ class CUDAGraphRunner:
1264 positions,
1265 kv_caches,
1266 attn_metadata,
1267 + intermediate_inputs,
1268 **kwargs,
1269 )
1270 torch.cuda.synchronize()
@@ -1220,18 +1272,27 @@ class CUDAGraphRunner:
1272 # Capture the graph.
1273 self._graph = torch.cuda.CUDAGraph()
1274 with torch.cuda.graph(self._graph, pool=memory_pool, stream=stream):
1223 - output_hidden_states = self.model(
1275 + output_hidden_or_intermediate_states = self.model(
1276 input_ids,
1277 positions,
1278 kv_caches,
1279 attn_metadata,
1280 + intermediate_inputs,
1281 **kwargs,
1282 )
1230 - if hidden_states is not None:
1231 - hidden_states.copy_(output_hidden_states)
1283 + if hidden_or_intermediate_states is not None:
1284 + if get_pp_group().is_last_rank:
1285 + hidden_or_intermediate_states.copy_(
1286 + output_hidden_or_intermediate_states)
1287 + else:
1288 + for key in hidden_or_intermediate_states.tensors:
1289 + hidden_or_intermediate_states[key].copy_(
1290 + output_hidden_or_intermediate_states[key])
1291 else:
1233 - hidden_states = output_hidden_states
1234 - del output_hidden_states
1292 + hidden_or_intermediate_states = (
1293 + output_hidden_or_intermediate_states)
1294 +
1295 + del output_hidden_or_intermediate_states
1296 # make sure `output_hidden_states` is deleted
1297 # in the graph's memory pool
1298 gc.collect()
@@ -1255,8 +1316,15 @@ class CUDAGraphRunner:
1316 attn_metadata.decode_metadata.seq_lens_tensor,
1317 "block_tables": attn_metadata.decode_metadata.block_tables,
1318 }
1258 - self.output_buffers = {"hidden_states": hidden_states}
1259 - return hidden_states
1319 + if intermediate_inputs is not None:
1320 + self.input_buffers.update(intermediate_inputs.tensors)
1321 + if get_pp_group().is_last_rank:
1322 + self.output_buffers = {
1323 + "hidden_states": hidden_or_intermediate_states
1324 + }
1325 + else:
1326 + self.output_buffers = hidden_or_intermediate_states
1327 + return hidden_or_intermediate_states
1328
1329 def forward(
1330 self,
@@ -1264,6 +1332,7 @@ class CUDAGraphRunner:
1332 positions: torch.Tensor,
1333 kv_caches: List[torch.Tensor],
1334 attn_metadata: AttentionMetadata,
1335 + intermediate_tensors: Optional[IntermediateTensors],
1336 **kwargs,
1337 ) -> torch.Tensor:
1338 # KV caches are fixed tensors, so we don't need to copy them.
@@ -1280,11 +1349,18 @@ class CUDAGraphRunner:
1349 non_blocking=True)
1350 self.input_buffers["block_tables"].copy_(
1351 attn_metadata.decode_metadata.block_tables, non_blocking=True)
1352 + if intermediate_tensors is not None:
1353 + for key in intermediate_tensors.tensors:
1354 + self.input_buffers[key].copy_(intermediate_tensors[key],
1355 + non_blocking=True)
1356 # Run the graph.
1357 self.graph.replay()
1358
1359 # Return the output tensor.
1287 - return self.output_buffers["hidden_states"]
1360 + if get_pp_group().is_last_rank:
1361 + return self.output_buffers["hidden_states"]
1362 +
1363 + return self.output_buffers
1364
1365 def __call__(self, *args, **kwargs):
1366 return self.forward(*args, **kwargs)
vllm/worker/model_runner_base.py
+4 -1
@@ -5,7 +5,8 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Type,
5
6 import torch
7
8 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
8 +from vllm.sequence import (IntermediateTensors, SamplerOutput,
9 + SequenceGroupMetadata)
10
11 if TYPE_CHECKING:
12 from vllm.attention import AttentionMetadata
@@ -137,6 +138,7 @@ class ModelRunnerBase(ABC, Generic[T]):
138 def prepare_model_input(
139 self,
140 seq_group_metadata_list: List[SequenceGroupMetadata],
141 + virtual_engine: int = 0,
142 ) -> T:
143 """
144 Prepare the inputs to ModelRunnerBase.execute_model from an execution
@@ -150,6 +152,7 @@ class ModelRunnerBase(ABC, Generic[T]):
152 self,
153 model_input: T,
154 kv_caches: Optional[List[torch.Tensor]],
155 + intermediate_tensors: Optional[IntermediateTensors],
156 num_steps: int = 1,
157 ) -> Optional[List[SamplerOutput]]:
158 """
vllm/worker/neuron_model_runner.py
+4 -1
@@ -9,7 +9,8 @@ from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
9 from vllm.logger import init_logger
10 from vllm.model_executor import SamplingMetadata
11 from vllm.model_executor.model_loader.neuron import get_neuron_model
12 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
12 +from vllm.sequence import (IntermediateTensors, SamplerOutput,
13 + SequenceGroupMetadata)
14 from vllm.utils import is_pin_memory_available, make_tensor_with_pad
15 from vllm.worker.model_runner_base import ModelRunnerBase, ModelRunnerInputBase
16
@@ -175,6 +176,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
176 def prepare_model_input(
177 self,
178 seq_group_metadata_list: List[SequenceGroupMetadata],
179 + virtual_engine: int = 0,
180 ) -> ModelInputForNeuron:
181 # NOTE: We assume that all sequences in the group are all prompts or
182 # all decodes.
@@ -207,6 +209,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
209 self,
210 model_input: ModelInputForNeuron,
211 kv_caches: Optional[List[torch.Tensor]] = None,
212 + intermediate_tensors: Optional[IntermediateTensors] = None,
213 num_steps: int = 1,
214 ) -> Optional[List[SamplerOutput]]:
215 if num_steps > 1:
vllm/worker/neuron_worker.py
+1 -1
@@ -80,7 +80,7 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
80 return False
81
82 @property
83 - def kv_cache(self) -> Optional[List[torch.Tensor]]:
83 + def kv_cache(self) -> Optional[List[List[torch.Tensor]]]:
84 return None
85
86 @torch.inference_mode()
vllm/worker/worker.py
+23 -13
@@ -59,9 +59,9 @@ class Worker(LocalOrDistributedWorkerBase):
59 self.lora_config = lora_config
60 self.load_config = load_config
61 self.is_driver_worker = is_driver_worker
62 - if self.is_driver_worker:
63 - assert self.rank == 0, "The driver worker must have rank 0."
64 -
62 + if parallel_config and is_driver_worker:
63 + assert rank % parallel_config.tensor_parallel_size == 0, \
64 + "Driver worker should be rank 0 of tensor parallel group."
65 if self.model_config.trust_remote_code:
66 # note: lazy import to avoid importing torch before initializing
67 from vllm.utils import init_cached_hf_modules
@@ -99,9 +99,9 @@ class Worker(LocalOrDistributedWorkerBase):
99 )
100 # Uninitialized cache engine. Will be initialized by
101 # initialize_cache.
102 - self.cache_engine: CacheEngine
102 + self.cache_engine: List[CacheEngine]
103 # Initialize gpu_cache as embedding models don't initialize kv_caches
104 - self.gpu_cache: Optional[List[torch.tensor]] = None
104 + self.gpu_cache: Optional[List[List[torch.tensor]]] = None
105
106 def init_device(self) -> None:
107 if self.device_config.device.type == "cuda":
@@ -217,10 +217,15 @@ class Worker(LocalOrDistributedWorkerBase):
217
218 def _init_cache_engine(self):
219 assert self.cache_config.num_gpu_blocks is not None
220 - self.cache_engine = CacheEngine(self.cache_config, self.model_config,
221 - self.parallel_config,
222 - self.device_config)
223 - self.gpu_cache = self.cache_engine.gpu_cache
220 + self.cache_engine = [
221 + CacheEngine(self.cache_config, self.model_config,
222 + self.parallel_config, self.device_config)
223 + for _ in range(self.parallel_config.pipeline_parallel_size)
224 + ]
225 + self.gpu_cache = [
226 + self.cache_engine[ve].gpu_cache
227 + for ve in range(self.parallel_config.pipeline_parallel_size)
228 + ]
229
230 def _warm_up_model(self) -> None:
231 if not self.model_config.enforce_eager:
@@ -234,12 +239,13 @@ class Worker(LocalOrDistributedWorkerBase):
239 return self.parallel_config.tensor_parallel_size > 1
240
241 @property
237 - def kv_cache(self) -> Optional[List[torch.Tensor]]:
242 + def kv_cache(self) -> Optional[List[List[torch.Tensor]]]:
243 return self.gpu_cache
244
245 @torch.inference_mode()
246 def prepare_worker_input(
247 self, execute_model_req: ExecuteModelRequest) -> WorkerInput:
248 + virtual_engine = execute_model_req.virtual_engine
249 num_seq_groups = len(execute_model_req.seq_group_metadata_list)
250 # `blocks_to_swap_in` and `blocks_to_swap_out` are cpu tensors.
251 # they contain parameters to launch cudamemcpyasync.
@@ -261,20 +267,24 @@ class Worker(LocalOrDistributedWorkerBase):
267 blocks_to_swap_in=blocks_to_swap_in,
268 blocks_to_swap_out=blocks_to_swap_out,
269 blocks_to_copy=blocks_to_copy,
270 + virtual_engine=virtual_engine,
271 )
272
273 @torch.inference_mode()
274 def execute_worker(self, worker_input: WorkerInput) -> None:
275 + virtual_engine = worker_input.virtual_engine
276 # Issue cache operations.
277 if (worker_input.blocks_to_swap_in is not None
278 and worker_input.blocks_to_swap_in.numel() > 0):
271 - self.cache_engine.swap_in(worker_input.blocks_to_swap_in)
279 + self.cache_engine[virtual_engine].swap_in(
280 + worker_input.blocks_to_swap_in)
281 if (worker_input.blocks_to_swap_out is not None
282 and worker_input.blocks_to_swap_out.numel() > 0):
274 - self.cache_engine.swap_out(worker_input.blocks_to_swap_out)
283 + self.cache_engine[virtual_engine].swap_out(
284 + worker_input.blocks_to_swap_out)
285 if (worker_input.blocks_to_copy is not None
286 and worker_input.blocks_to_copy.numel() > 0):
277 - self.cache_engine.copy(worker_input.blocks_to_copy)
287 + self.cache_engine[virtual_engine].copy(worker_input.blocks_to_copy)
288
289 def add_lora(self, lora_request: LoRARequest) -> bool:
290 return self.model_runner.add_lora(lora_request)
vllm/worker/worker_base.py
+31 -9
@@ -6,10 +6,11 @@ from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union
6
7 import torch
8
9 -from vllm.distributed import broadcast_tensor_dict
9 +from vllm.distributed import broadcast_tensor_dict, get_pp_group
10 from vllm.logger import init_logger
11 from vllm.lora.request import LoRARequest
12 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
12 +from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
13 + SamplerOutput)
14 from vllm.utils import (enable_trace_function_call_for_thread, is_hip,
15 update_environment_variables)
16 from vllm.worker.model_runner_base import ModelRunnerBase, ModelRunnerInputBase
@@ -124,6 +125,7 @@ class WorkerInput:
125 blocks_to_swap_in: Optional[torch.Tensor] = None
126 blocks_to_swap_out: Optional[torch.Tensor] = None
127 blocks_to_copy: Optional[torch.Tensor] = None
128 + virtual_engine: int = 0
129
130 @classmethod
131 def from_broadcasted_tensor_dict(
@@ -139,6 +141,7 @@ class WorkerInput:
141 blocks_to_swap_in=tensor_dict.pop("blocks_to_swap_in"),
142 blocks_to_swap_out=tensor_dict.pop("blocks_to_swap_out"),
143 blocks_to_copy=tensor_dict.pop("blocks_to_copy"),
144 + virtual_engine=tensor_dict["virtual_engine"],
145 )
146
147 def as_broadcastable_tensor_dict(
@@ -151,6 +154,7 @@ class WorkerInput:
154 "blocks_to_swap_in": self.blocks_to_swap_in,
155 "blocks_to_swap_out": self.blocks_to_swap_out,
156 "blocks_to_copy": self.blocks_to_copy,
157 + "virtual_engine": self.virtual_engine,
158 }
159
160 return tensor_dict
@@ -181,11 +185,13 @@ class LocalOrDistributedWorkerBase(WorkerBase):
185
186 @property
187 @abstractmethod
184 - def kv_cache(self) -> Optional[List[torch.Tensor]]:
188 + def kv_cache(self) -> Optional[List[List[torch.Tensor]]]:
189 """
186 - Get the kv cache to pass to the worker's model runner. Used by the
187 - default `execute_model`. If the worker's model runner does not follow
188 - the ModelRunnerBase interface, then inherit from WorkerBase instead.
190 + Gets the list of kv caches to pass to the worker's model runner. Each
191 + element in the list is a kv cache corresponding to a particular virtual
192 + engine (PP stream). Used by the default `execute_model`. If the worker's
193 + model runner does not follow the ModelRunnerBase interface, then inherit
194 + from WorkerBase instead.
195 """
196 raise NotImplementedError
197
@@ -227,7 +233,8 @@ class LocalOrDistributedWorkerBase(WorkerBase):
233 execute_model_req=execute_model_req)
234 model_input: ModelRunnerInputBase = (
235 self.model_runner.prepare_model_input(
230 - execute_model_req.seq_group_metadata_list))
236 + execute_model_req.seq_group_metadata_list,
237 + execute_model_req.virtual_engine))
238 num_steps = execute_model_req.num_steps
239
240 if self.do_metadata_broadcast:
@@ -255,8 +262,23 @@ class LocalOrDistributedWorkerBase(WorkerBase):
262 if worker_input.num_seq_groups == 0:
263 return []
264
258 - return self.model_runner.execute_model(model_input, self.kv_cache,
259 - num_steps)
265 + intermediate_tensors = None
266 + if not get_pp_group().is_first_rank:
267 + intermediate_tensors = IntermediateTensors(
268 + get_pp_group().recv_tensor_dict())
269 +
270 + output = self.model_runner.execute_model(
271 + model_input, self.kv_cache[worker_input.virtual_engine]
272 + if self.kv_cache is not None else None, intermediate_tensors,
273 + num_steps)
274 +
275 + if not get_pp_group().is_last_rank:
276 + get_pp_group().send_tensor_dict(output.tensors)
277 + return [None]
278 +
279 + # Worker only supports single-step execution. Wrap the output in a
280 + # list to conform to interface.
281 + return output
282
283
284 class WorkerWrapperBase:
vllm/worker/xpu_model_runner.py
+4 -1
@@ -12,7 +12,8 @@ from vllm.distributed import broadcast_tensor_dict
12 from vllm.logger import init_logger
13 from vllm.model_executor.model_loader import get_model
14 from vllm.sampling_params import SamplingParams
15 -from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
15 +from vllm.sequence import (IntermediateTensors, SamplerOutput, SequenceData,
16 + SequenceGroupMetadata)
17 from vllm.utils import CudaMemoryProfiler, make_tensor_with_pad
18 from vllm.worker.model_runner import AttentionMetadata, SamplingMetadata
19 from vllm.worker.model_runner_base import (
@@ -190,6 +191,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
191 def prepare_model_input(
192 self,
193 seq_group_metadata_list: List[SequenceGroupMetadata],
194 + virtual_engine: int = 0,
195 ) -> ModelInputForXPU:
196 multi_modal_input = None
197 if self.is_driver_worker:
@@ -334,6 +336,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
336 self,
337 model_input: ModelInputForXPU,
338 kv_caches: List[torch.Tensor],
339 + intermediate_tensors: Optional[IntermediateTensors] = None,
340 num_steps: int = 1,
341 ) -> Optional[List[SamplerOutput]]:
342 if num_steps > 1:
vllm/worker/xpu_worker.py
+2 -2
@@ -85,8 +85,8 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
85 )
86 # Uninitialized cache engine. Will be initialized by
87 # initialize_cache.
88 - self.cache_engine: CacheEngine
89 - self.gpu_cache: List[torch.Tensor]
88 + self.cache_engine: List[CacheEngine]
89 + self.gpu_cache: Optional[List[List[torch.Tensor]]]
90
91 def init_device(self) -> None:
92 if self.device_config.device.type == "xpu" and is_xpu():