@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 67b4221a61
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core][5/N] Fully working chunked prefill e2e (#3884)

SangBin Cho committed Apr 11, 2024 at 09:56 UTC 67b4221a61ace91a79aff507df0a95a01978300e
26 files changed +927 -315
.buildkite/test-pipeline.yaml
+2
@@ -29,6 +29,8 @@ steps:
29 - pytest -v -s test_pynccl.py
30 - TEST_DIST_MODEL=facebook/opt-125m pytest -v -s test_basic_distributed_correctness.py
31 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s test_basic_distributed_correctness.py
32 + - TEST_DIST_MODEL=facebook/opt-125m pytest -v -s test_chunked_prefill_distributed.py
33 + - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s test_chunked_prefill_distributed.py
34
35 - label: Engine Test
36 command: pytest -v -s engine tokenization test_sequence.py test_config.py
benchmarks/benchmark_latency.py
+1 -2
@@ -177,8 +177,7 @@ if __name__ == '__main__':
177 help='block size of key/value cache')
178 parser.add_argument(
179 '--enable-chunked-prefill',
180 - type=bool,
181 - default=False,
180 + action='store_true',
181 help='If True, the prefill requests can be chunked based on the '
182 'max_num_batched_tokens')
183 parser.add_argument(
benchmarks/benchmark_throughput.py
+38 -24
@@ -74,25 +74,31 @@ def run_vllm(
74 quantization_param_path: Optional[str],
75 device: str,
76 enable_prefix_caching: bool,
77 + enable_chunked_prefill: bool,
78 + max_num_batched_tokens: int,
79 gpu_memory_utilization: float = 0.9,
80 download_dir: Optional[str] = None,
81 ) -> float:
82 from vllm import LLM, SamplingParams
81 - llm = LLM(model=model,
82 - tokenizer=tokenizer,
83 - quantization=quantization,
84 - tensor_parallel_size=tensor_parallel_size,
85 - seed=seed,
86 - trust_remote_code=trust_remote_code,
87 - dtype=dtype,
88 - max_model_len=max_model_len,
89 - gpu_memory_utilization=gpu_memory_utilization,
90 - enforce_eager=enforce_eager,
91 - kv_cache_dtype=kv_cache_dtype,
92 - quantization_param_path=quantization_param_path,
93 - device=device,
94 - enable_prefix_caching=enable_prefix_caching,
95 - download_dir=download_dir)
83 + llm = LLM(
84 + model=model,
85 + tokenizer=tokenizer,
86 + quantization=quantization,
87 + tensor_parallel_size=tensor_parallel_size,
88 + seed=seed,
89 + trust_remote_code=trust_remote_code,
90 + dtype=dtype,
91 + max_model_len=max_model_len,
92 + gpu_memory_utilization=gpu_memory_utilization,
93 + enforce_eager=enforce_eager,
94 + kv_cache_dtype=kv_cache_dtype,
95 + quantization_param_path=quantization_param_path,
96 + device=device,
97 + enable_prefix_caching=enable_prefix_caching,
98 + download_dir=download_dir,
99 + enable_chunked_prefill=enable_chunked_prefill,
100 + max_num_batched_tokens=max_num_batched_tokens,
101 + )
102
103 # Add the requests to the engine.
104 for prompt, _, output_len in requests:
@@ -213,15 +219,15 @@ def main(args: argparse.Namespace):
219 args.output_len)
220
221 if args.backend == "vllm":
216 - elapsed_time = run_vllm(requests, args.model, args.tokenizer,
217 - args.quantization, args.tensor_parallel_size,
218 - args.seed, args.n, args.use_beam_search,
219 - args.trust_remote_code, args.dtype,
220 - args.max_model_len, args.enforce_eager,
221 - args.kv_cache_dtype,
222 - args.quantization_param_path, args.device,
223 - args.enable_prefix_caching,
224 - args.gpu_memory_utilization, args.download_dir)
222 + elapsed_time = run_vllm(
223 + requests, args.model, args.tokenizer, args.quantization,
224 + args.tensor_parallel_size, args.seed, args.n, args.use_beam_search,
225 + args.trust_remote_code, args.dtype, args.max_model_len,
226 + args.enforce_eager, args.kv_cache_dtype,
227 + args.quantization_param_path, args.device,
228 + args.enable_prefix_caching, args.enable_chunked_prefill,
229 + args.max_num_batched_tokens, args.gpu_memory_utilization,
230 + args.download_dir)
231 elif args.backend == "hf":
232 assert args.tensor_parallel_size == 1
233 elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -335,6 +341,14 @@ if __name__ == "__main__":
341 "--enable-prefix-caching",
342 action='store_true',
343 help="enable automatic prefix caching for vLLM backend.")
344 + parser.add_argument("--enable-chunked-prefill",
345 + action='store_true',
346 + help="enable chunked prefill for vLLM backend.")
347 + parser.add_argument('--max-num-batched-tokens',
348 + type=int,
349 + default=None,
350 + help='maximum number of batched tokens per '
351 + 'iteration')
352 parser.add_argument('--download-dir',
353 type=str,
354 default=None,
tests/basic_correctness/test_chunked_prefill.py new
+70
@@ -0,0 +1,70 @@
1 +"""Compare the outputs of HF and vLLM when using greedy sampling.
2 +
3 +It tests chunked prefill. Chunked prefill can be enabled by
4 +enable_chunked_prefill=True. If prefill size exceeds max_num_batched_tokens,
5 +prefill requests are chunked.
6 +
7 +Run `pytest tests/models/test_chunked_prefill.py`.
8 +"""
9 +import pytest
10 +
11 +MODELS = [
12 + "facebook/opt-125m",
13 + "meta-llama/Llama-2-7b-hf",
14 +]
15 +
16 +
17 +@pytest.mark.parametrize("model", MODELS)
18 +@pytest.mark.parametrize("dtype", ["half"])
19 +@pytest.mark.parametrize("max_tokens", [32])
20 +@pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16])
21 +@pytest.mark.parametrize("enforce_eager", [False, True])
22 +# NOTE: Increasing this in this suite will fail CI because we currently cannot
23 +# reset distributed env properly. Use a value > 1 just when you test.
24 +@pytest.mark.parametrize("tensor_parallel_size", [1])
25 +def test_models(
26 + hf_runner,
27 + vllm_runner,
28 + example_prompts,
29 + model: str,
30 + dtype: str,
31 + max_tokens: int,
32 + chunked_prefill_token_size: int,
33 + enforce_eager: bool,
34 + tensor_parallel_size: int,
35 +) -> None:
36 + if (tensor_parallel_size == 2 and chunked_prefill_token_size != 16
37 + and not enforce_eager):
38 + pytest.skip(f"Skip {chunked_prefill_token_size=} and {enforce_eager=} "
39 + "for high TP to save testing time.")
40 + max_num_seqs = min(chunked_prefill_token_size, 256)
41 + enable_chunked_prefill = False
42 + max_num_batched_tokens = None
43 + if chunked_prefill_token_size != -1:
44 + enable_chunked_prefill = True
45 + max_num_batched_tokens = chunked_prefill_token_size
46 +
47 + hf_model = hf_runner(model, dtype=dtype)
48 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
49 + del hf_model
50 +
51 + vllm_model = vllm_runner(
52 + model,
53 + dtype=dtype,
54 + max_num_batched_tokens=max_num_batched_tokens,
55 + enable_chunked_prefill=enable_chunked_prefill,
56 + tensor_parallel_size=tensor_parallel_size,
57 + enforce_eager=enforce_eager,
58 + max_num_seqs=max_num_seqs,
59 + )
60 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
61 + del vllm_model
62 + print(vllm_outputs[0])
63 +
64 + for i in range(len(example_prompts)):
65 + hf_output_ids, hf_output_str = hf_outputs[i]
66 + vllm_output_ids, vllm_output_str = vllm_outputs[i]
67 + assert hf_output_str == vllm_output_str, (
68 + f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
69 + assert hf_output_ids == vllm_output_ids, (
70 + f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
tests/core/test_chunked_prefill_scheduler.py
+8 -8
@@ -104,10 +104,10 @@ def test_chunk():
104 # One chunked prefill, and one decoding.
105 seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
106 assert set(get_sequence_groups(out)) == set(running)
107 - # The first one is decoding.
108 - assert seq_group_meta[0].token_chunk_size == 1
107 + # The first one is prefill. Scheduler guarantees ordering.
108 + assert seq_group_meta[0].token_chunk_size == 56
109 # The second one is a chunked prefill.
110 - assert seq_group_meta[1].token_chunk_size == 56
110 + assert seq_group_meta[1].token_chunk_size == 1
111 assert out.num_prefill_groups == 1
112 assert out.num_batched_tokens == 57
113
@@ -157,12 +157,12 @@ def test_complex():
157 # Decoding & chunked prefill & first chunk of 3rd request is scheduled.
158 seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
159 assert len(get_sequence_groups(out)) == 3
160 - # The first one is decoding.
161 - assert seq_group_meta[0].token_chunk_size == 1
162 - # The second one is a chunked prefill.
160 + # The first one is the first chunked prefill.
161 + assert seq_group_meta[0].token_chunk_size == 7
162 + # The second one is the second new chunked prefill.
163 assert seq_group_meta[1].token_chunk_size == 56
164 - # The third one is also chunked.
165 - assert seq_group_meta[2].token_chunk_size == 7
164 + # The last one is decode.
165 + assert seq_group_meta[2].token_chunk_size == 1
166 # Two of them are in chunked prefill.
167 assert out.num_prefill_groups == 2
168 assert out.num_batched_tokens == 64
tests/distributed/test_basic_distributed_correctness.py
+6 -1
@@ -33,11 +33,16 @@ def test_models(
33 dtype: str,
34 max_tokens: int,
35 ) -> None:
36 +
37 hf_model = hf_runner(model, dtype=dtype)
38 hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
39 del hf_model
40
40 - vllm_model = vllm_runner(model, dtype=dtype, tensor_parallel_size=2)
41 + vllm_model = vllm_runner(
42 + model,
43 + dtype=dtype,
44 + tensor_parallel_size=2,
45 + )
46 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
47 del vllm_model
48
tests/distributed/test_chunked_prefill_distributed.py new
+66
@@ -0,0 +1,66 @@
1 +"""Compare the outputs of HF and distributed vLLM when using greedy sampling.
2 +vLLM will allocate all the available memory, so we need to run the tests one
3 +by one. The solution is to pass arguments (model name) by environment
4 +variables.
5 +
6 +Run:
7 +```sh
8 +TEST_DIST_MODEL=facebook/opt-125m pytest \
9 + test_chunked_prefill_distributed.py
10 +TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf \
11 + test_chunked_prefill_distributed.py
12 +```
13 +"""
14 +import os
15 +
16 +import pytest
17 +import torch
18 +
19 +MODELS = [
20 + os.environ["TEST_DIST_MODEL"],
21 +]
22 +
23 +
24 +@pytest.mark.skipif(torch.cuda.device_count() < 2,
25 + reason="Need at least 2 GPUs to run the test.")
26 +@pytest.mark.parametrize("model", MODELS)
27 +@pytest.mark.parametrize("dtype", ["half"])
28 +@pytest.mark.parametrize("max_tokens", [5])
29 +@pytest.mark.parametrize("chunked_prefill_token_size", [16])
30 +def test_models(
31 + hf_runner,
32 + vllm_runner,
33 + example_prompts,
34 + model: str,
35 + dtype: str,
36 + max_tokens: int,
37 + chunked_prefill_token_size: int,
38 +) -> None:
39 + # Add a chunked prefill config.
40 + max_num_seqs = min(chunked_prefill_token_size, 256)
41 + assert chunked_prefill_token_size != -1
42 + enable_chunked_prefill = True
43 + max_num_batched_tokens = chunked_prefill_token_size
44 +
45 + hf_model = hf_runner(model, dtype=dtype)
46 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
47 + del hf_model
48 +
49 + vllm_model = vllm_runner(
50 + model,
51 + dtype=dtype,
52 + tensor_parallel_size=2,
53 + max_num_seqs=max_num_seqs,
54 + enable_chunked_prefill=enable_chunked_prefill,
55 + max_num_batched_tokens=max_num_batched_tokens,
56 + )
57 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
58 + del vllm_model
59 +
60 + for i in range(len(example_prompts)):
61 + hf_output_ids, hf_output_str = hf_outputs[i]
62 + vllm_output_ids, vllm_output_str = vllm_outputs[i]
63 + assert hf_output_str == vllm_output_str, (
64 + f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
65 + assert hf_output_ids == vllm_output_ids, (
66 + f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
tests/entrypoints/test_openai_server.py
+1 -1
@@ -141,7 +141,7 @@ def server(zephyr_lora_files):
141 "--max-cpu-loras",
142 "2",
143 "--max-num-seqs",
144 - "128"
144 + "128",
145 ])
146 ray.get(server_runner.ready.remote())
147 yield server_runner
tests/models/test_models.py
+1 -1
@@ -12,7 +12,7 @@ MODELS = [
12 "gpt2",
13 "bigcode/tiny_starcoder_py",
14 "EleutherAI/pythia-70m",
15 - "bigscience/bloom-560m",
15 + "bigscience/bloom-560m", # Testing alibi slopes.
16 "microsoft/phi-2",
17 "stabilityai/stablelm-3b-4e1t",
18 # "allenai/OLMo-1B", # Broken
tests/worker/test_model_runner.py
+170 -19
@@ -1,14 +1,18 @@
1 import pytest
2 import torch
3
4 -from vllm.config import ModelConfig
4 +from vllm.config import ModelConfig, SchedulerConfig
5 from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
6 from vllm.worker.model_runner import ModelRunner, _get_graph_batch_size
7
8
9 @pytest.mark.parametrize("batch_size", list(range(1, 257)))
10 def test_prepare_prompt(batch_size):
11 - model_runner = ModelRunner(None, None, None, None, None)
11 + scheduler_config = SchedulerConfig(100000,
12 + 100000,
13 + 100000,
14 + enable_chunked_prefill=False)
15 + model_runner = ModelRunner(None, None, scheduler_config, None, None)
16 model_runner.set_block_size(16)
17
18 prompt_lens = []
@@ -36,8 +40,10 @@ def test_prepare_prompt(batch_size):
40 prompt_len - 1)
41 selected_token_start_idx += prompt_len
42 (input_tokens, input_positions, attn_metadata, return_prompt_lens, _, _, _,
39 - _, _) = (model_runner._prepare_prompt(seq_group_metadata_list))
43 + _, _,
44 + slot_mapping) = (model_runner._prepare_prompt(seq_group_metadata_list))
45 assert return_prompt_lens == prompt_lens
46 + assert len(slot_mapping) == len(input_tokens)
47
48 # Verify input metadata is correct for prompts.
49 device = model_runner.device
@@ -45,8 +51,6 @@ def test_prepare_prompt(batch_size):
51 assert torch.allclose(attn_metadata.prompt_lens_tensor,
52 torch.tensor(prompt_lens, device=device))
53 assert attn_metadata.prompt_lens == prompt_lens
48 - assert attn_metadata.num_prompt_tokens == sum(prompt_lens)
49 - assert attn_metadata.num_generation_tokens == 0
54 assert attn_metadata.max_prompt_len == max(prompt_lens)
55
56 # Test subquery start locs.
@@ -83,23 +87,22 @@ def test_prepare_prompt(batch_size):
87 assert torch.allclose(attn_metadata.block_tables, expected)
88 # Cuda graph should not be used for prerill.
89 assert attn_metadata.use_cuda_graph is False
86 - assert attn_metadata.kv_cache_dtype == "auto"
90
88 - assert input_tokens.shape == (sum(prompt_lens), )
89 - assert input_positions.shape == (sum(prompt_lens), )
91 + assert len(input_tokens) == sum(prompt_lens)
92 + assert len(input_positions) == sum(prompt_lens)
93 torch.testing.assert_close(input_tokens, input_positions)
94
95 sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
96 prompt_lens,
97 subquery_lens=prompt_lens)
95 - assert input_tokens.shape == (sum(prompt_lens), )
96 - assert input_positions.shape == (sum(prompt_lens), )
98 + assert len(input_tokens) == sum(prompt_lens)
99 + assert len(input_positions) == sum(prompt_lens)
100 actual = sampling_metadata.selected_token_indices
101 expected = torch.tensor(expected_selected_token_indices,
102 device=actual.device,
103 dtype=actual.dtype)
104 torch.testing.assert_close(actual, expected)
102 - torch.testing.assert_close(input_tokens, input_positions)
105 + assert input_tokens == input_positions
106
107 actual = sampling_metadata.selected_token_indices
108 expected = torch.tensor(expected_selected_token_indices,
@@ -122,7 +125,12 @@ def test_prepare_decode_cuda_graph(batch_size):
125 revision=None,
126 enforce_eager=False,
127 )
125 - model_runner = ModelRunner(model_config, None, None, None, None)
128 + scheduler_config = SchedulerConfig(100000,
129 + 100000,
130 + 100000,
131 + enable_chunked_prefill=False)
132 + model_runner = ModelRunner(model_config, None, scheduler_config, None,
133 + None)
134 model_runner.set_block_size(16)
135
136 prompt_lens = []
@@ -143,16 +151,15 @@ def test_prepare_decode_cuda_graph(batch_size):
151 assert seq_group_metadata.token_chunk_size == 1
152 seq_group_metadata_list.append(seq_group_metadata)
153
146 - input_tokens, input_positions, attn_metadata, _, _, _ = (
154 + input_tokens, input_positions, attn_metadata, _, _, _, slot_mapping = (
155 model_runner._prepare_decode(seq_group_metadata_list))
156 + assert len(slot_mapping) == len(input_tokens)
157
158 expected_bs = _get_graph_batch_size(len(seq_group_metadata_list))
159 # Verify input metadata is correct for prompts.
160 device = model_runner.device
161 assert attn_metadata.is_prompt is False
162 assert attn_metadata.prompt_lens is None
154 - assert attn_metadata.num_prompt_tokens == 0
155 - assert attn_metadata.num_generation_tokens == expected_bs
163 assert attn_metadata.max_prompt_len is None
164 assert attn_metadata.subquery_start_loc is None
165 assert attn_metadata.seq_start_loc is None
@@ -170,11 +177,10 @@ def test_prepare_decode_cuda_graph(batch_size):
177 model_runner.get_max_block_per_batch())
178 # Cuda graph should not be used for prerill.
179 assert attn_metadata.use_cuda_graph is True
173 - assert attn_metadata.kv_cache_dtype == "auto"
180
175 - assert input_tokens.shape == (expected_bs, )
176 - assert input_positions.shape == (expected_bs, )
177 - torch.testing.assert_close(input_tokens, input_positions)
181 + assert len(input_tokens) == expected_bs
182 + assert len(input_positions) == expected_bs
183 + assert input_tokens == input_positions
184
185 # Verify Sampling
186 expected_selected_token_indices = []
@@ -190,3 +196,148 @@ def test_prepare_decode_cuda_graph(batch_size):
196 device=actual.device,
197 dtype=actual.dtype)
198 torch.testing.assert_close(actual, expected)
199 +
200 +
201 +def test_empty_seq_group():
202 + """Verify prepare prompt and decode returns empty output."""
203 + model_config = ModelConfig(
204 + "facebook/opt-125m",
205 + "facebook/opt-125m",
206 + tokenizer_mode="auto",
207 + trust_remote_code=False,
208 + download_dir=None,
209 + load_format="dummy",
210 + seed=0,
211 + dtype="float16",
212 + revision=None,
213 + enforce_eager=False,
214 + )
215 + model_runner = ModelRunner(model_config, None, None, None, None)
216 + model_runner.set_block_size(16)
217 + seq_group_metadata_list = []
218 + input_tokens, input_positions, attn_metadata, _, _, _, slot_mapping = (
219 + model_runner._prepare_decode(seq_group_metadata_list))
220 + assert len(input_tokens) == 0
221 + assert len(input_positions) == 0
222 + assert attn_metadata is None
223 + assert len(slot_mapping) == 0
224 +
225 + (input_tokens, input_positions, attn_metadata, return_prompt_lens, _, _, _,
226 + _, _,
227 + slot_mapping) = (model_runner._prepare_prompt(seq_group_metadata_list))
228 + assert len(input_tokens) == 0
229 + assert len(input_positions) == 0
230 + assert attn_metadata is None
231 + assert len(slot_mapping) == 0
232 + assert len(return_prompt_lens) == 0
233 +
234 +
235 +@pytest.mark.parametrize("batch_size", list(range(2, 128)))
236 +@pytest.mark.parametrize("enforce_eager", [True, False])
237 +def test_hybrid_batches(batch_size, enforce_eager, monkeypatch):
238 +
239 + def get_world_size(group=None):
240 + return 1
241 +
242 + def mock_get_process_group_ranks(group=None):
243 + return [0]
244 +
245 + monkeypatch.setattr(torch.distributed, "get_world_size", get_world_size)
246 + monkeypatch.setattr(torch.distributed, "get_process_group_ranks",
247 + mock_get_process_group_ranks)
248 +
249 + model_config = ModelConfig(
250 + "facebook/opt-125m",
251 + "facebook/opt-125m",
252 + tokenizer_mode="auto",
253 + trust_remote_code=False,
254 + download_dir=None,
255 + load_format="dummy",
256 + seed=0,
257 + dtype="float16",
258 + revision=None,
259 + enforce_eager=enforce_eager,
260 + )
261 + scheduler_config = SchedulerConfig(100000,
262 + 100000,
263 + 100000,
264 + enable_chunked_prefill=True)
265 + model_runner = ModelRunner(model_config,
266 + None,
267 + scheduler_config,
268 + None,
269 + None,
270 + is_driver_worker=True)
271 + model_runner.set_block_size(16)
272 +
273 + # Add prefill requests.
274 + prompt_lens = []
275 + seq_group_metadata_list = []
276 + prefill_metadata_list = []
277 + decode_metadata_list = []
278 + block_tables = {0: [1]}
279 + prefill_batch_size = batch_size // 2
280 + decode_batch_size = batch_size - prefill_batch_size
281 + for i in range(prefill_batch_size):
282 + # make sure all tokens fit into one block
283 + prompt_len = i % (model_runner.block_size - 1) + 1
284 + prompt_lens.append(prompt_len)
285 + seq_data = SequenceData(list(range(prompt_len)))
286 + seq_group_metadata = SequenceGroupMetadata(
287 + request_id=f"test_{i}",
288 + is_prompt=True,
289 + seq_data={0: seq_data},
290 + sampling_params=SamplingParams(temperature=0),
291 + block_tables=block_tables,
292 + )
293 + assert seq_group_metadata.token_chunk_size == seq_data.get_len()
294 + seq_group_metadata_list.append(seq_group_metadata)
295 + prefill_metadata_list.append(seq_group_metadata)
296 +
297 + # Add decode requests
298 + for i in range(prefill_batch_size, batch_size):
299 + # make sure all tokens fit into one block
300 + prompt_len = i % (model_runner.block_size - 1) + 1
301 + prompt_toks = list(range(prompt_len))
302 + seq_data = SequenceData(prompt_toks)
303 + seq_group_metadata = SequenceGroupMetadata(
304 + request_id=f"test_{i}",
305 + is_prompt=False,
306 + seq_data={0: seq_data},
307 + sampling_params=SamplingParams(temperature=0),
308 + block_tables={0: [1]},
309 + )
310 + assert seq_group_metadata.token_chunk_size == 1
311 + seq_group_metadata_list.append(seq_group_metadata)
312 + decode_metadata_list.append(seq_group_metadata)
313 +
314 + (input_tokens, input_positions, attn_metadata, _, _, _,
315 + _) = model_runner.prepare_input_tensors(seq_group_metadata_list)
316 +
317 + prefill_meta_actual = attn_metadata.prefill_metadata
318 + decode_meta_actual = attn_metadata.decode_metadata
319 +
320 + assert len(attn_metadata.slot_mapping) == len(input_tokens)
321 + assert len(input_positions) == len(input_tokens)
322 + assert attn_metadata.kv_cache_dtype == "auto"
323 + assert attn_metadata.num_prefills == prefill_batch_size
324 + if enforce_eager:
325 + assert attn_metadata.num_decode_tokens == decode_batch_size
326 + else:
327 + assert attn_metadata.num_decode_tokens == _get_graph_batch_size(
328 + decode_batch_size)
329 + assert attn_metadata.num_prefill_tokens == sum(prompt_lens)
330 +
331 + # Verify attn metadata is consistent. We don't need to test individual
332 + # values here because they are tested above.
333 + prefill_meta = model_runner._prepare_prompt(
334 + prefill_metadata_list).attn_metadata
335 + decode_meta = model_runner._prepare_decode(
336 + decode_metadata_list).attn_metadata
337 +
338 + for attr_expected, attr_actual in zip(vars(prefill_meta),
339 + vars(prefill_meta_actual)):
340 + assert attr_expected[1] == attr_actual[1]
341 + for attr_expected, attr_actual in zip(vars(decode_meta),
342 + vars(decode_meta_actual)):
343 + assert attr_expected[1] == attr_actual[1]
vllm/attention/__init__.py
+3 -1
@@ -1,5 +1,6 @@
1 from vllm.attention.backends.abstract import (AttentionBackend,
2 - AttentionMetadata)
2 + AttentionMetadata,
3 + AttentionMetadataPerStage)
4 from vllm.attention.layer import Attention
5 from vllm.attention.selector import get_attn_backend
6
@@ -8,4 +9,5 @@ __all__ = [
9 "AttentionMetadata",
10 "Attention",
11 "get_attn_backend",
12 + "AttentionMetadataPerStage",
13 ]
vllm/attention/backends/abstract.py
+39 -3
@@ -1,6 +1,6 @@
1 from abc import ABC, abstractmethod
2 from dataclasses import dataclass, fields
3 -from typing import Any, Dict, List, Optional, Tuple, Type
3 +from typing import Any, Dict, Generic, List, Optional, Tuple, Type, TypeVar
4
5 import torch
6
@@ -47,7 +47,8 @@ class AttentionBackend(ABC):
47
48
49 @dataclass
50 -class AttentionMetadata:
50 +class AttentionMetadataPerStage:
51 + """Attention metadata for a specific stage. I.e., prefill or decode."""
52
53 def asdict_zerocopy(self) -> Dict[str, Any]:
54 """Similar to dataclasses.asdict, but avoids deepcopying."""
@@ -59,6 +60,41 @@ class AttentionMetadata:
60 }
61
62
63 +T = TypeVar("T", bound=AttentionMetadataPerStage)
64 +
65 +
66 +@dataclass
67 +class AttentionMetadata(Generic[T]):
68 + """Attention metadata for prefill and decode batched together."""
69 + # Total number of prefill requests.
70 + num_prefills: int
71 + # Number of prefill tokens.
72 + num_prefill_tokens: int
73 + # Number of decode tokens. Note that it is equivalent to the number of
74 + # decode requests.
75 + num_decode_tokens: int
76 + # The attention metadata for prefill requests in a batch.
77 + # None if there's no prefill requests in a batch.
78 + prefill_metadata: Optional[T]
79 + # The attention metadata for decode requests in a batch.
80 + # None if there's no decode requests in a batch.
81 + decode_metadata: Optional[T]
82 + # (num_tokens,). The indices of the token slots that input tokens will be
83 + # stored into. E.g., if `slot_mapping` is [35, 2, 17] and the block size
84 + # is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot
85 + # in block 0, and 1st slot in block 1, respectively.
86 + slot_mapping: torch.Tensor
87 + # The kv cache's data type.
88 + kv_cache_dtype: str
89 +
90 + def __post_init__(self):
91 + if self.num_prefill_tokens > 0:
92 + assert self.num_prefills > 0
93 + assert self.prefill_metadata is not None
94 + if self.num_decode_tokens > 0:
95 + assert self.decode_metadata is not None
96 +
97 +
98 class AttentionImpl(ABC):
99
100 @abstractmethod
@@ -80,7 +116,7 @@ class AttentionImpl(ABC):
116 key: torch.Tensor,
117 value: torch.Tensor,
118 kv_cache: torch.Tensor,
83 - attn_metadata: AttentionMetadata,
119 + attn_metadata: AttentionMetadata[AttentionMetadataPerStage],
120 kv_scale: float,
121 ) -> torch.Tensor:
122 raise NotImplementedError
vllm/attention/backends/flash_attn.py
+55 -30
@@ -11,7 +11,8 @@ import torch
11 from flash_attn import flash_attn_varlen_func
12
13 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14 - AttentionMetadata)
14 + AttentionMetadata,
15 + AttentionMetadataPerStage)
16 from vllm.attention.ops.paged_attn import (PagedAttention,
17 PagedAttentionMetadata)
18
@@ -53,7 +54,8 @@ class FlashAttentionBackend(AttentionBackend):
54
55
56 @dataclass
56 -class FlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
57 +class FlashAttentionMetadata(AttentionMetadataPerStage,
58 + PagedAttentionMetadata):
59 """Metadata for FlashAttentionBackend.
60
61 NOTE: Any python object stored here is not updated when it is
@@ -68,10 +70,6 @@ class FlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
70 prompt_lens: Optional[List[int]]
71 # prompt_lens stored as a tensor.
72 prompt_lens_tensor: Optional[torch.Tensor]
71 - # The number of prompt tokens. Doesn't include padding.
72 - num_prompt_tokens: int
73 - # The number of generation tokens. Doesn't include padding.
74 - num_generation_tokens: int
73
74 # NOTE(sang): Definition of context_len, subquery_len, and seqlen.
75 # |---------- N-1 iteration --------|
@@ -107,18 +105,27 @@ class FlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
105 class FlashAttentionImpl(AttentionImpl):
106 """
107 If the input tensors contain prompt tokens, the layout is as follows:
110 - |<--------------- num_prompt_tokens -------------->|
111 - |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->|
108 + |<--------------- num_prefill_tokens ----------------->|
109 + |<--prefill_0-->|<--prefill_1-->|...|<--prefill_N-1--->|
110
111 Otherwise, the layout is as follows:
114 - |<------------------ num_generation_tokens (M) ----------------->|
115 - |<--generation_0-->|..........|<--generation_M-1-->|<--padding-->|
112 + |<----------------- num_decode_tokens ------------------>|
113 + |<--decode_0-->|..........|<--decode_M-1-->|<--padding-->|
114
115 Generation tokens can contain padding when cuda-graph is used.
116 Currently, prompt tokens don't contain any padding.
117
118 The prompts might have different lengths, while the generation tokens
119 always have length 1.
120 +
121 + If chunked prefill is enabled, prefill tokens and decode tokens can be
122 + batched together in a flattened 1D query.
123 +
124 + |<----- num_prefill_tokens ---->|<------- num_decode_tokens --------->|
125 + |<-prefill_0->|...|<-prefill_N-1->|<--decode_0-->|...|<--decode_M-1-->|
126 +
127 + Currently, cuda graph is disabled for chunked prefill, meaning there's no
128 + padding between prefill and decode tokens.
129 """
130
131 def __init__(
@@ -155,7 +162,7 @@ class FlashAttentionImpl(AttentionImpl):
162 key: torch.Tensor,
163 value: torch.Tensor,
164 kv_cache: torch.Tensor,
158 - attn_metadata: FlashAttentionMetadata,
165 + attn_metadata: AttentionMetadata[FlashAttentionMetadata],
166 kv_scale: float,
167 ) -> torch.Tensor:
168 """Forward pass with FlashAttention and PagedAttention.
@@ -188,52 +195,70 @@ class FlashAttentionImpl(AttentionImpl):
195 attn_metadata.kv_cache_dtype,
196 kv_scale)
197
191 - if attn_metadata.is_prompt:
198 + num_prefill_tokens = attn_metadata.num_prefill_tokens
199 + num_decode_tokens = attn_metadata.num_decode_tokens
200 + assert key.shape[0] == num_prefill_tokens + num_decode_tokens
201 + assert value.shape[0] == num_prefill_tokens + num_decode_tokens
202 +
203 + output = torch.empty_like(query)
204 + # Query for decode. KV is not needed because it is already cached.
205 + decode_query = query[num_prefill_tokens:]
206 + # QKV for prefill.
207 + query = query[:num_prefill_tokens]
208 + key = key[:num_prefill_tokens]
209 + value = value[:num_prefill_tokens]
210 +
211 + assert query.shape[0] == num_prefill_tokens
212 + assert decode_query.shape[0] == num_decode_tokens
213 +
214 + if prefill_meta := attn_metadata.prefill_metadata:
215 # Prompt run.
193 - if kv_cache is None or attn_metadata.block_tables.numel() == 0:
216 + if kv_cache is None or prefill_meta.block_tables.numel() == 0:
217 # normal attention
218 # When block_tables are not filled, it means q and k are the
219 # prompt, and they have the same length.
197 - output = flash_attn_varlen_func(
220 + out = flash_attn_varlen_func(
221 q=query,
222 k=key,
223 v=value,
201 - cu_seqlens_q=attn_metadata.seq_start_loc,
202 - cu_seqlens_k=attn_metadata.seq_start_loc,
203 - max_seqlen_q=attn_metadata.max_prompt_len,
204 - max_seqlen_k=attn_metadata.max_prompt_len,
224 + cu_seqlens_q=prefill_meta.seq_start_loc,
225 + cu_seqlens_k=prefill_meta.seq_start_loc,
226 + max_seqlen_q=prefill_meta.max_prompt_len,
227 + max_seqlen_k=prefill_meta.max_prompt_len,
228 softmax_scale=self.scale,
229 causal=True,
230 window_size=self.sliding_window,
231 alibi_slopes=self.alibi_slopes,
232 )
233 + assert output[:num_prefill_tokens].shape == out.shape
234 + output[:num_prefill_tokens] = out
235 else:
236 # prefix-enabled attention
237 # TODO(Hai) this triton kernel has regression issue (broke) to
238 # deal with different data types between KV and FP8 KV cache,
239 # to be addressed separately.
215 - output = PagedAttention.forward_prefix(
240 + output[:num_prefill_tokens] = PagedAttention.forward_prefix(
241 query,
242 key,
243 value,
244 key_cache,
245 value_cache,
221 - attn_metadata.block_tables,
222 - attn_metadata.subquery_start_loc,
223 - attn_metadata.prompt_lens_tensor,
224 - attn_metadata.context_lens,
225 - attn_metadata.max_subquery_len,
246 + prefill_meta.block_tables,
247 + prefill_meta.subquery_start_loc,
248 + prefill_meta.prompt_lens_tensor,
249 + prefill_meta.context_lens,
250 + prefill_meta.max_subquery_len,
251 self.alibi_slopes,
252 )
228 - else:
253 + if decode_meta := attn_metadata.decode_metadata:
254 # Decoding run.
230 - output = PagedAttention.forward_decode(
231 - query,
255 + output[num_prefill_tokens:] = PagedAttention.forward_decode(
256 + decode_query,
257 key_cache,
258 value_cache,
234 - attn_metadata.block_tables,
235 - attn_metadata.context_lens,
236 - attn_metadata.max_context_len,
259 + decode_meta.block_tables,
260 + decode_meta.context_lens,
261 + decode_meta.max_context_len,
262 attn_metadata.kv_cache_dtype,
263 self.num_kv_heads,
264 self.scale,
vllm/attention/backends/rocm_flash_attn.py
+63 -34
@@ -6,7 +6,8 @@ from typing import Dict, List, Optional, Tuple, Type
6 import torch
7
8 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
9 - AttentionMetadata)
9 + AttentionMetadata,
10 + AttentionMetadataPerStage)
11 from vllm.attention.ops.paged_attn import (PagedAttention,
12 PagedAttentionMetadata)
13 from vllm.logger import init_logger
@@ -51,7 +52,8 @@ class ROCmFlashAttentionBackend(AttentionBackend):
52
53
54 @dataclass
54 -class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
55 +class ROCmFlashAttentionMetadata(AttentionMetadataPerStage,
56 + PagedAttentionMetadata):
57 """Metadata for FlashAttentionBackend.
58
59 NOTE: Any python object stored here is not updated when it is
@@ -66,10 +68,6 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
68 prompt_lens: Optional[List[int]]
69 # prompt_lens stored as a tensor.
70 prompt_lens_tensor: Optional[torch.Tensor]
69 - # The number of prompt tokens. Doesn't include padding.
70 - num_prompt_tokens: int
71 - # The number of generation tokens. Doesn't include padding.
72 - num_generation_tokens: int
71
72 # NOTE(sang): Definition of context_len, subquery_len, and seqlen.
73 # |---------- N-1 iteration --------|
@@ -117,6 +115,15 @@ class ROCmFlashAttentionImpl(AttentionImpl):
115
116 The prompts might have different lengths, while the generation tokens
117 always have length 1.
118 +
119 + If chunked prefill is enabled, prefill tokens and decode tokens can be
120 + batched together in a flattened 1D query.
121 +
122 + |<----- num_prefill_tokens ---->|<------- num_decode_tokens ----------->|
123 + |<-prompt_0->|...|<-prompt_N-1->|<-generation_0->|...|<-generation_M-1->|
124 +
125 + Currently, cuda graph is disabled for chunked prefill, meaning there's no
126 + padding between prefill and decode tokens.
127 """
128
129 def __init__(
@@ -181,7 +188,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
188 key: torch.Tensor,
189 value: torch.Tensor,
190 kv_cache: torch.Tensor,
184 - attn_metadata: ROCmFlashAttentionMetadata,
191 + attn_metadata: AttentionMetadata[ROCmFlashAttentionMetadata],
192 kv_scale: float = 1.0,
193 ) -> torch.Tensor:
194 """Forward pass with FlashAttention and PagedAttention.
@@ -218,9 +225,25 @@ class ROCmFlashAttentionImpl(AttentionImpl):
225 kv_scale,
226 )
227
221 - if attn_metadata.is_prompt:
228 + num_prefill_tokens = attn_metadata.num_prefill_tokens
229 + num_decode_tokens = attn_metadata.num_decode_tokens
230 + assert key.shape[0] == num_prefill_tokens + num_decode_tokens
231 + assert value.shape[0] == num_prefill_tokens + num_decode_tokens
232 +
233 + output = torch.empty_like(query)
234 + # Query for decode. KV is not needed because it is already cached.
235 + decode_query = query[num_prefill_tokens:]
236 + # QKV for prefill.
237 + query = query[:num_prefill_tokens]
238 + key = key[:num_prefill_tokens]
239 + value = value[:num_prefill_tokens]
240 +
241 + assert query.shape[0] == num_prefill_tokens
242 + assert decode_query.shape[0] == num_decode_tokens
243 +
244 + if prefill_meta := attn_metadata.prefill_metadata:
245 # Prompt run.
223 - if kv_cache is None or attn_metadata.block_tables.numel() == 0:
246 + if kv_cache is None or prefill_meta.block_tables.numel() == 0:
247 # triton attention
248 # When block_tables are not filled, it means q and k are the
249 # prompt, and they have the same length.
@@ -230,63 +253,69 @@ class ROCmFlashAttentionImpl(AttentionImpl):
253 key = self.repeat_kv(key, self.num_queries_per_kv)
254 value = self.repeat_kv(value, self.num_queries_per_kv)
255 if self.use_naive_attn:
233 - output = self.attn_fuc(
256 + out = self.attn_fuc(
257 query,
258 key,
259 value,
237 - attn_metadata.prompt_lens,
260 + prefill_meta.prompt_lens,
261 self.scale,
262 )
263 + assert output[:num_prefill_tokens].shape == out.shape
264 + output[:num_prefill_tokens] = out
265 else:
241 - output, _ = self.attn_func(
266 + out, _ = self.attn_func(
267 query,
268 key,
269 value,
270 None,
246 - attn_metadata.seq_start_loc,
247 - attn_metadata.seq_start_loc,
248 - attn_metadata.max_prompt_len,
249 - attn_metadata.max_prompt_len,
271 + prefill_meta.seq_start_loc,
272 + prefill_meta.seq_start_loc,
273 + prefill_meta.max_prompt_len,
274 + prefill_meta.max_prompt_len,
275 True,
276 self.scale,
277 )
278 + assert output[:num_prefill_tokens].shape == out.shape
279 + output[:num_prefill_tokens] = out
280 else:
254 - output = self.attn_func(
281 + out = self.attn_func(
282 q=query,
283 k=key,
284 v=value,
258 - cu_seqlens_q=attn_metadata.seq_start_loc,
259 - cu_seqlens_k=attn_metadata.seq_start_loc,
260 - max_seqlen_q=attn_metadata.max_prompt_len,
261 - max_seqlen_k=attn_metadata.max_prompt_len,
285 + cu_seqlens_q=prefill_meta.seq_start_loc,
286 + cu_seqlens_k=prefill_meta.seq_start_loc,
287 + max_seqlen_q=prefill_meta.max_prompt_len,
288 + max_seqlen_k=prefill_meta.max_prompt_len,
289 softmax_scale=self.scale,
290 causal=True,
291 )
265 -
292 + assert output[:num_prefill_tokens].shape == out.shape
293 + output[:num_prefill_tokens] = out
294 else:
295 # prefix-enabled attention
268 - output = PagedAttention.forward_prefix(
296 + output[:num_prefill_tokens] = PagedAttention.forward_prefix(
297 query,
298 key,
299 value,
300 key_cache,
301 value_cache,
274 - attn_metadata.block_tables,
275 - attn_metadata.subquery_start_loc,
276 - attn_metadata.prompt_lens_tensor,
277 - attn_metadata.context_lens,
278 - attn_metadata.max_subquery_len,
302 + prefill_meta.block_tables,
303 + prefill_meta.subquery_start_loc,
304 + prefill_meta.prompt_lens_tensor,
305 + prefill_meta.context_lens,
306 + prefill_meta.max_subquery_len,
307 self.alibi_slopes,
308 )
281 - else:
309 +
310 + if decode_meta := attn_metadata.decode_metadata:
311 # Decoding run.
283 - output = PagedAttention.forward_decode(
284 - query,
312 + output[num_prefill_tokens:] = PagedAttention.forward_decode(
313 + decode_query,
314 key_cache,
315 value_cache,
287 - attn_metadata.block_tables,
288 - attn_metadata.context_lens,
289 - attn_metadata.max_context_len,
316 + decode_meta.block_tables,
317 + decode_meta.context_lens,
318 + decode_meta.max_context_len,
319 attn_metadata.kv_cache_dtype,
320 self.num_kv_heads,
321 self.scale,
vllm/attention/backends/torch_sdpa.py
+42 -25
@@ -7,7 +7,8 @@ import torch
7 from torch.nn.functional import scaled_dot_product_attention
8
9 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10 - AttentionMetadata)
10 + AttentionMetadata,
11 + AttentionMetadataPerStage)
12 from vllm.attention.ops.paged_attn import (PagedAttention,
13 PagedAttentionMetadata)
14
@@ -49,17 +50,14 @@ class TorchSDPABackend(AttentionBackend):
50
51
52 @dataclass
52 -class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
53 +class TorchSDPAMetadata(AttentionMetadataPerStage, PagedAttentionMetadata):
54 """Metadata for TorchSDPABackend.
55 """
56 # Currently, input sequences can only contain all prompts
57 # or all decoding. True if all sequences are prompts.
58 is_prompt: bool
58 - slot_mapping: torch.Tensor
59 prompt_lens: Optional[List[int]]
60 prompt_lens_tensor: Optional[torch.Tensor]
61 - num_prompt_tokens: int
62 - num_generation_tokens: int
61
62 max_subquery_len: Optional[int] = None
63 max_prompt_len: Optional[int] = None
@@ -113,7 +111,7 @@ class TorchSDPABackendImpl(AttentionImpl):
111 key: torch.Tensor,
112 value: torch.Tensor,
113 kv_cache: Optional[torch.Tensor],
116 - attn_metadata: TorchSDPAMetadata,
114 + attn_metadata: AttentionMetadata[TorchSDPAMetadata],
115 kv_scale: float,
116 ) -> torch.Tensor:
117 """Forward pass with torch SDPA and PagedAttention.
@@ -142,36 +140,51 @@ class TorchSDPABackendImpl(AttentionImpl):
140 attn_metadata.kv_cache_dtype,
141 kv_scale)
142
145 - if attn_metadata.is_prompt:
146 - if (kv_cache is None or attn_metadata.block_tables.numel() == 0):
143 + num_prefill_tokens = attn_metadata.num_prefill_tokens
144 + num_decode_tokens = attn_metadata.num_decode_tokens
145 + assert key.shape[0] == num_prefill_tokens + num_decode_tokens
146 + assert value.shape[0] == num_prefill_tokens + num_decode_tokens
147 +
148 + output = torch.empty_like(query)
149 + # Query for decode. KV is not needed because it is already cached.
150 + decode_query = query[num_prefill_tokens:]
151 + # QKV for prefill.
152 + query = query[:num_prefill_tokens]
153 + key = key[:num_prefill_tokens]
154 + value = value[:num_prefill_tokens]
155 +
156 + assert query.shape[0] == num_prefill_tokens
157 + assert decode_query.shape[0] == num_decode_tokens
158 +
159 + if prefill_meta := attn_metadata.prefill_metadata:
160 + if (kv_cache is None or prefill_meta.block_tables.numel() == 0):
161 if self.num_kv_heads != self.num_heads:
162 key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
163 value = value.repeat_interleave(self.num_queries_per_kv,
164 dim=1)
165
152 - if attn_metadata.attn_bias is None:
166 + if prefill_meta.attn_bias is None:
167 if self.alibi_slopes is not None:
168 att_masks = _make_alibi_bias(
169 self.alibi_slopes, query.dtype,
156 - attn_metadata.prompt_lens) # type: ignore
170 + prefill_meta.prompt_lens) # type: ignore
171 elif self.sliding_window is not None:
172 att_masks = _make_sliding_window_bias(
159 - attn_metadata.prompt_lens, self.sliding_window,
173 + prefill_meta.prompt_lens, self.sliding_window,
174 query.dtype) # type: ignore
175 else:
162 - att_masks = [None] * len(attn_metadata.prompt_lens)
163 - attn_metadata.attn_bias = att_masks
176 + att_masks = [None] * len(prefill_meta.prompt_lens)
177 + prefill_meta.attn_bias = att_masks
178
179 query = query.movedim(0, query.dim() - 2)
180 key = key.movedim(0, key.dim() - 2)
181 value = value.movedim(0, value.dim() - 2)
182
183 start = 0
170 - output = torch.empty(
171 - (num_tokens, self.num_heads, self.head_size),
172 - dtype=query.dtype)
173 - for prompt_len, mask in zip(attn_metadata.prompt_lens,
174 - attn_metadata.attn_bias):
184 + out = torch.empty((num_tokens, self.num_heads, self.head_size),
185 + dtype=query.dtype)
186 + for prompt_len, mask in zip(prefill_meta.prompt_lens,
187 + prefill_meta.attn_bias):
188 end = start + prompt_len
189 sub_out = scaled_dot_product_attention(
190 query[:, start:end, :],
@@ -181,28 +194,32 @@ class TorchSDPABackendImpl(AttentionImpl):
194 dropout_p=0.0,
195 is_causal=not self.need_mask,
196 scale=self.scale).movedim(query.dim() - 2, 0)
184 - output[start:end, :, :] = sub_out
197 + out[start:end, :, :] = sub_out
198 start = end
199 + assert out.shape == output[:num_prefill_tokens].shape
200 + output[:num_prefill_tokens] = out
201 else:
202 # prefix-enabled attention
203 raise RuntimeError(
204 "Torch SDPA backend doesn't support prefix decoding.")
205
191 - else:
206 + if decode_meta := attn_metadata.decode_metadata:
207 # Decoding run.
193 - output = PagedAttention.forward_decode(
194 - query,
208 + out = PagedAttention.forward_decode(
209 + decode_query,
210 key_cache,
211 value_cache,
197 - attn_metadata.block_tables,
198 - attn_metadata.context_lens,
199 - attn_metadata.max_context_len,
212 + decode_meta.block_tables,
213 + decode_meta.context_lens,
214 + decode_meta.max_context_len,
215 attn_metadata.kv_cache_dtype,
216 self.num_kv_heads,
217 self.scale,
218 self.alibi_slopes,
219 kv_scale,
220 )
221 + assert out.shape == output[num_prefill_tokens:].shape
222 + output[num_prefill_tokens:]
223
224 # Reshape the output tensor.
225 return output.view(-1, self.num_heads * self.head_size)
vllm/attention/backends/xformers.py
+79 -59
@@ -9,7 +9,8 @@ from xformers.ops.fmha.attn_bias import (AttentionBias,
9 LowerTriangularMaskWithTensorBias)
10
11 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
12 - AttentionMetadata)
12 + AttentionMetadata,
13 + AttentionMetadataPerStage)
14 from vllm.attention.ops.paged_attn import (PagedAttention,
15 PagedAttentionMetadata)
16 from vllm.logger import init_logger
@@ -54,7 +55,7 @@ class XFormersBackend(AttentionBackend):
55
56
57 @dataclass
57 -class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
58 +class XFormersMetadata(AttentionMetadataPerStage, PagedAttentionMetadata):
59 """Metadata for XFormersbackend.
60
61 NOTE: Any python object stored here is not updated when it is
@@ -65,19 +66,10 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
66 # Currently, input sequences can only contain all prompts
67 # or all decoding. True if all sequences are prompts.
68 is_prompt: bool
68 - # (num_tokens,). The indices of the token slots that input tokens will be
69 - # stored into. E.g., if `slot_mapping` is [35, 2, 17] and the block size
70 - # is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot
71 - # in block 0, and 1st slot in block 1, respectively.
72 - slot_mapping: torch.Tensor
69 # (batch_size,). The prompt length per sequence. None if it is a decoding.
70 prompt_lens: Optional[List[int]]
71 # prompt_lens stored as a tensor.
72 prompt_lens_tensor: Optional[torch.Tensor]
77 - # The number of prompt tokens. Doesn't include padding.
78 - num_prompt_tokens: int
79 - # The number of generation tokens. Doesn't include padding.
80 - num_generation_tokens: int
73
74 # NOTE(sang): Definition of context_len, subquery_len, and seqlen.
75 # |---------- N-1 iteration --------|
@@ -123,18 +115,27 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
115 class XFormersImpl(AttentionImpl):
116 """
117 If the input tensors contain prompt tokens, the layout is as follows:
126 - |<--------------- num_prompt_tokens --------------->|
127 - |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1--->|
118 + |<--------------- num_prefill_tokens ----------------->|
119 + |<--prefill_0-->|<--prefill_1-->|...|<--prefill_N-1--->|
120
121 Otherwise, the layout is as follows:
130 - |<------------------ num_generation_tokens (M) ----------------->|
131 - |<--generation_0-->|..........|<--generation_M-1-->|<--padding-->|
122 + |<----------------- num_decode_tokens ------------------>|
123 + |<--decode_0-->|..........|<--decode_M-1-->|<--padding-->|
124
125 Generation tokens can contain padding when cuda-graph is used.
126 Currently, prompt tokens don't contain any padding.
127
128 The prompts might have different lengths, while the generation tokens
129 always have length 1.
130 +
131 + If chunked prefill is enabled, prefill tokens and decode tokens can be
132 + batched together in a flattened 1D query.
133 +
134 + |<----- num_prefill_tokens ---->|<------- num_decode_tokens --------->|
135 + |<-prefill_0->|...|<-prefill_N-1->|<--decode_0-->|...|<--decode_M-1-->|
136 +
137 + Currently, cuda graph is disabled for chunked prefill, meaning there's no
138 + padding between prefill and decode tokens.
139 """
140
141 def __init__(
@@ -170,7 +171,7 @@ class XFormersImpl(AttentionImpl):
171 key: torch.Tensor,
172 value: torch.Tensor,
173 kv_cache: Optional[torch.Tensor],
173 - attn_metadata: XFormersMetadata,
174 + attn_metadata: AttentionMetadata[XFormersMetadata],
175 kv_scale: float,
176 ) -> torch.Tensor:
177 """Forward pass with xFormers and PagedAttention.
@@ -202,59 +203,61 @@ class XFormersImpl(AttentionImpl):
203 attn_metadata.kv_cache_dtype,
204 kv_scale)
205
205 - if attn_metadata.is_prompt:
206 + num_prefill_tokens = attn_metadata.num_prefill_tokens
207 + num_decode_tokens = attn_metadata.num_decode_tokens
208 + assert key.shape[0] == num_prefill_tokens + num_decode_tokens
209 + assert value.shape[0] == num_prefill_tokens + num_decode_tokens
210 +
211 + output = torch.empty_like(query)
212 + # Query for decode. KV is not needed because it is already cached.
213 + decode_query = query[num_prefill_tokens:]
214 + # QKV for prefill.
215 + query = query[:num_prefill_tokens]
216 + key = key[:num_prefill_tokens]
217 + value = value[:num_prefill_tokens]
218 +
219 + assert query.shape[0] == num_prefill_tokens
220 + assert decode_query.shape[0] == num_decode_tokens
221 +
222 + if prefill_meta := attn_metadata.prefill_metadata:
223 # Prompt run.
207 - if kv_cache is None or attn_metadata.block_tables.numel() == 0:
224 + if kv_cache is None or prefill_meta.block_tables.numel() == 0:
225 # normal attention.
226 # block tables are empty if the prompt does not have a cached
227 # prefix.
211 - if self.num_kv_heads != self.num_heads:
212 - # As of Nov 2023, xformers only supports MHA. For MQA/GQA,
213 - # project the key and value tensors to the desired number of
214 - # heads.
215 - # TODO(woosuk): Use MQA/GQA kernels for higher performance.
216 - query = query.view(query.shape[0], self.num_kv_heads,
217 - self.num_queries_per_kv,
218 - query.shape[-1])
219 - key = key[:, :,
220 - None, :].expand(key.shape[0], self.num_kv_heads,
221 - self.num_queries_per_kv,
222 - key.shape[-1])
223 - value = value[:, :,
224 - None, :].expand(value.shape[0],
225 - self.num_kv_heads,
226 - self.num_queries_per_kv,
227 - value.shape[-1])
228 -
229 - output = self._run_memory_efficient_xformers_forward(
230 - query, key, value, attn_metadata)
228 + out = self._run_memory_efficient_xformers_forward(
229 + query, key, value, prefill_meta)
230 + assert out.shape == output[:num_prefill_tokens].shape
231 + output[:num_prefill_tokens] = out
232 else:
233 # prefix-enabled attention
234 # TODO(Hai) this triton kernel has regression issue (broke) to
235 # deal with different data types between KV and FP8 KV cache,
236 # to be addressed separately.
236 - output = PagedAttention.forward_prefix(
237 + out = PagedAttention.forward_prefix(
238 query,
239 key,
240 value,
241 key_cache,
242 value_cache,
242 - attn_metadata.block_tables,
243 - attn_metadata.subquery_start_loc,
244 - attn_metadata.prompt_lens_tensor,
245 - attn_metadata.context_lens,
246 - attn_metadata.max_subquery_len,
243 + prefill_meta.block_tables,
244 + prefill_meta.subquery_start_loc,
245 + prefill_meta.prompt_lens_tensor,
246 + prefill_meta.context_lens,
247 + prefill_meta.max_subquery_len,
248 self.alibi_slopes,
249 )
249 - else:
250 - # Decoding run.
251 - output = PagedAttention.forward_decode(
252 - query,
250 + assert output[:num_prefill_tokens].shape == out.shape
251 + output[:num_prefill_tokens] = out
252 +
253 + if decode_meta := attn_metadata.decode_metadata:
254 + output[num_prefill_tokens:] = PagedAttention.forward_decode(
255 + decode_query,
256 key_cache,
257 value_cache,
255 - attn_metadata.block_tables,
256 - attn_metadata.context_lens,
257 - attn_metadata.max_context_len,
258 + decode_meta.block_tables,
259 + decode_meta.context_lens,
260 + decode_meta.max_context_len,
261 attn_metadata.kv_cache_dtype,
262 self.num_kv_heads,
263 self.scale,
@@ -275,13 +278,30 @@ class XFormersImpl(AttentionImpl):
278 """Attention for 1D query of multiple prompts. Multiple prompt
279 tokens are flattened in to `query` input.
280
281 + See https://facebookresearch.github.io/xformers/components/ops.html
282 + for API spec.
283 +
284 Args:
279 - output: shape = [num_prompt_tokens, num_heads, head_size]
280 - query: shape = [num_prompt_tokens, num_heads, head_size]
281 - key: shape = [num_prompt_tokens, num_kv_heads, head_size]
282 - value: shape = [num_prompt_tokens, num_kv_heads, head_size]
285 + output: shape = [num_prefill_tokens, num_heads, head_size]
286 + query: shape = [num_prefill_tokens, num_heads, head_size]
287 + key: shape = [num_prefill_tokens, num_kv_heads, head_size]
288 + value: shape = [num_prefill_tokens, num_kv_heads, head_size]
289 attn_metadata: Metadata for attention.
290 """
291 + original_query = query
292 + if self.num_kv_heads != self.num_heads:
293 + # GQA/MQA requires the shape [B, M, G, H, K].
294 + # Note that the output also has the same shape (which is different
295 + # from a spec from the doc).
296 + query = query.view(query.shape[0], self.num_kv_heads,
297 + self.num_queries_per_kv, query.shape[-1])
298 + key = key[:, :,
299 + None, :].expand(key.shape[0], self.num_kv_heads,
300 + self.num_queries_per_kv, key.shape[-1])
301 + value = value[:, :,
302 + None, :].expand(value.shape[0], self.num_kv_heads,
303 + self.num_queries_per_kv,
304 + value.shape[-1])
305 # Set attention bias if not provided. This typically happens at
306 # the very attention layer of every iteration.
307 # FIXME(woosuk): This is a hack.
@@ -302,6 +322,7 @@ class XFormersImpl(AttentionImpl):
322 # TODO(woosuk): Too many view operations. Let's try to reduce
323 # them in the future for code readability.
324 if self.alibi_slopes is None:
325 + # Add the batch dimension.
326 query = query.unsqueeze(0)
327 key = key.unsqueeze(0)
328 value = value.unsqueeze(0)
@@ -312,14 +333,13 @@ class XFormersImpl(AttentionImpl):
333 attn_bias=attn_metadata.attn_bias[0],
334 p=0.0,
335 scale=self.scale)
315 -
316 - return out.view_as(query)
336 + return out.view_as(original_query)
337
338 # Attention with alibi slopes.
339 # FIXME(woosuk): Because xformers does not support dynamic sequence
340 # lengths with custom attention bias, we process each prompt one by
341 # one. This is inefficient, especially when we have many short prompts.
322 - output = torch.empty_like(query)
342 + output = torch.empty_like(original_query)
343 start = 0
344 for i, prompt_len in enumerate(attn_metadata.prompt_lens):
345 end = start + prompt_len
@@ -331,7 +351,7 @@ class XFormersImpl(AttentionImpl):
351 p=0.0,
352 scale=self.scale)
353 # TODO(woosuk): Unnecessary copy. Optimize.
334 - output[start:end].copy_(out.squeeze(0))
354 + output[start:end].copy_(out.view_as(original_query[start:end]))
355 start += prompt_len
356 return output
357
vllm/attention/layer.py
+3 -2
@@ -4,7 +4,8 @@ from typing import List, Optional
4 import torch
5 import torch.nn as nn
6
7 -from vllm.attention.backends.abstract import AttentionMetadata
7 +from vllm.attention.backends.abstract import (AttentionMetadata,
8 + AttentionMetadataPerStage)
9 from vllm.attention.selector import get_attn_backend
10
11
@@ -41,7 +42,7 @@ class Attention(nn.Module):
42 key: torch.Tensor,
43 value: torch.Tensor,
44 kv_cache: Optional[torch.Tensor],
44 - attn_metadata: AttentionMetadata,
45 + attn_metadata: AttentionMetadata[AttentionMetadataPerStage],
46 kv_scale: float = 1.0,
47 ) -> torch.Tensor:
48 return self.impl.forward(query, key, value, kv_cache, attn_metadata,
vllm/attention/ops/paged_attn.py
-6
@@ -13,11 +13,6 @@ _PARTITION_SIZE = 512
13 @dataclass
14 class PagedAttentionMetadata:
15 """Metadata for PagedAttention."""
16 - # (num_tokens,). The indices of the token slots that input tokens will be
17 - # stored into. E.g., if `slot_mapping` is [35, 2, 17] and the block size
18 - # is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot
19 - # in block 0, and 1st slot in block 1, respectively.
20 - slot_mapping: torch.Tensor
16 # (batch_size,). The length of context (tokens stored in KV cache) per
17 # sequence. WARNING: When it is a prefill request, it doesn't include new
18 # tokens. When it is for decoding, it includes a new token.
@@ -31,7 +26,6 @@ class PagedAttentionMetadata:
26 # 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph
27 # captured.
28 block_tables: Optional[torch.Tensor]
34 - kv_cache_dtype: str
29
30
31 class PagedAttention:
vllm/config.py
+10 -3
@@ -565,9 +565,16 @@ class SchedulerConfig:
565 if max_num_batched_tokens is not None:
566 self.max_num_batched_tokens = max_num_batched_tokens
567 else:
568 - # If max_model_len is too short, use 2048 as the default value for
569 - # higher throughput.
570 - self.max_num_batched_tokens = max(max_model_len, 2048)
568 + if enable_chunked_prefill:
569 + # For chunked prefill, choose the well-tuned batch size.
570 + self.max_num_batched_tokens = 768
571 + else:
572 + # If max_model_len is too short, use 2048 as the default value
573 + # for higher throughput.
574 + self.max_num_batched_tokens = max(max_model_len, 2048)
575 + if enable_chunked_prefill:
576 + logger.info("Chunked prefill is enabled (EXPERIMENTAL).")
577 +
578 self.max_num_seqs = max_num_seqs
579 self.max_model_len = max_model_len
580 self.use_v2_block_manager = use_v2_block_manager
vllm/core/scheduler.py
+9 -6
@@ -140,7 +140,11 @@ class SchedulerOutputs:
140
141 @property
142 def lora_requests(self) -> Set[LoRARequest]:
143 - return {g.seq_group.lora_request for g in self.scheduled_seq_groups}
143 + return {
144 + g.seq_group.lora_request
145 + for g in self.scheduled_seq_groups
146 + if g.seq_group.lora_request is not None
147 + }
148
149
150 @dataclass
@@ -826,13 +830,12 @@ class Scheduler:
830 # Update swapped requests.
831 self.swapped = remaining_swapped
832 self.swapped.extend(running_scheduled.swapped_out)
829 -
833 return SchedulerOutputs(
834 scheduled_seq_groups=(prefills.seq_groups +
832 - running_scheduled.decode_seq_groups +
835 running_scheduled.prefill_seq_groups +
834 - swapped_in.decode_seq_groups +
835 - swapped_in.prefill_seq_groups),
836 + swapped_in.prefill_seq_groups +
837 + running_scheduled.decode_seq_groups +
838 + swapped_in.decode_seq_groups),
839 num_prefill_groups=(len(prefills.seq_groups) +
840 len(swapped_in.prefill_seq_groups) +
841 len(running_scheduled.prefill_seq_groups)),
@@ -907,7 +910,7 @@ class Scheduler:
910
911 # It assumes the scheduled_seq_groups is ordered by
912 # prefill < decoding.
910 - is_prompt = i < scheduler_outputs.num_prefill_groups
913 + is_prompt = seq_group.is_prefill()
914 seq_group_metadata = SequenceGroupMetadata(
915 request_id=seq_group.request_id,
916 is_prompt=is_prompt,
vllm/distributed/communication_op.py
+9 -1
@@ -173,10 +173,18 @@ def broadcast_tensor_dict(
173 torch.distributed.broadcast_object_list([metadata_list],
174 src=src,
175 group=group)
176 + async_handles = []
177 for key, value in metadata_list:
178 if isinstance(value, TensorMetadata):
179 tensor = tensor_dict[key]
179 - torch.distributed.broadcast(tensor, src=src, group=group)
180 + async_handles.append(
181 + torch.distributed.broadcast(tensor,
182 + src=src,
183 + group=group,
184 + async_op=True))
185 + for async_handle in async_handles:
186 + async_handle.wait()
187 +
188 else:
189 recv_metadata_list = [None]
190 torch.distributed.broadcast_object_list(recv_metadata_list,
vllm/engine/arg_utils.py
+2 -3
@@ -386,9 +386,8 @@ class EngineArgs:
386 'prompt latency) before scheduling next prompt.')
387 parser.add_argument(
388 '--enable-chunked-prefill',
389 - type=bool,
390 - default=False,
391 - help='If True, the prefill requests can be chunked based on the '
389 + action='store_true',
390 + help='If set, the prefill requests can be chunked based on the '
391 'max_num_batched_tokens')
392
393 parser.add_argument(
vllm/engine/llm_engine.py
+4 -1
@@ -633,7 +633,10 @@ class LLMEngine:
633 seq_group = scheduled_seq_group.seq_group
634 seq_group.update_num_computed_tokens(
635 scheduled_seq_group.token_chunk_size)
636 - self._process_sequence_group_outputs(seq_group, outputs)
636 + # If uncomputed tokens > 0, it means prefill is chunked.
637 + # We don't need to process outputs in that case.
638 + if seq_group.get_num_uncomputed_tokens() == 0:
639 + self._process_sequence_group_outputs(seq_group, outputs)
640
641 # Free the finished sequence groups.
642 self.scheduler.free_finished_seq_groups()
vllm/lora/layers.py
+3 -2
@@ -267,12 +267,13 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
267
268 def forward(self, x: torch.Tensor) -> torch.Tensor:
269 added_tokens_mask = x > self.base_layer.org_vocab_size - 1
270 - indices = self.embeddings_indices[1][:self.indices_len[3]].view_as(x)
270 + embedding_len = self.indices_len[3]
271 + indices = self.embeddings_indices[1][:embedding_len].view_as(x)
272 full_lora_a_embeddings = F.embedding(
273 x + indices,
274 self.lora_a_stacked_2d,
275 )
275 - indices = self.embeddings_indices[0][:self.indices_len[3]].view_as(x)
276 + indices = self.embeddings_indices[0][:embedding_len].view_as(x)
277 full_output = self.base_layer.forward(
278 x.add_(indices * added_tokens_mask))
279
vllm/sequence.py
+2 -1
@@ -500,7 +500,8 @@ class SequenceGroup:
500 def get_num_uncomputed_tokens(self) -> int:
501 num_uncomputed_tokens = 0
502 for seq in self.get_seqs():
503 - num_uncomputed_tokens += seq.data.get_num_uncomputed_tokens()
503 + if not seq.is_finished():
504 + num_uncomputed_tokens += seq.data.get_num_uncomputed_tokens()
505 return num_uncomputed_tokens
506
507 def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
vllm/worker/model_runner.py
+241 -82
@@ -1,12 +1,14 @@
1 import contextlib
2 import time
3 -from typing import Dict, List, Optional, Set, Tuple
3 +from enum import IntEnum
4 +from typing import Dict, List, NamedTuple, Optional, Set, Tuple
5
6 import numpy as np
7 import torch
8 import torch.nn as nn
9
9 -from vllm.attention import AttentionMetadata, get_attn_backend
10 +from vllm.attention import (AttentionMetadata, AttentionMetadataPerStage,
11 + get_attn_backend)
12 from vllm.config import (DeviceConfig, LoRAConfig, ModelConfig, ParallelConfig,
13 SchedulerConfig, VisionLanguageConfig)
14 from vllm.distributed import broadcast_tensor_dict, with_pynccl_for_all_reduce
@@ -37,6 +39,66 @@ _BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
39 ]
40
41
42 +class PreparePromptMetadata(NamedTuple):
43 + input_tokens: List[int]
44 + input_positions: List[int]
45 + attn_metadata: Optional[AttentionMetadataPerStage]
46 + prompt_lens: List[int]
47 + subquery_lens: List[int]
48 + lora_index_mapping: List[int]
49 + lora_prompt_mapping: List[int]
50 + lora_requests: Set[LoRARequest]
51 + multi_modal_input: Optional[torch.Tensor]
52 + slot_mapping: List[int]
53 +
54 + @classmethod
55 + def empty(cls):
56 + return PreparePromptMetadata(
57 + input_tokens=[],
58 + input_positions=[],
59 + attn_metadata=None,
60 + prompt_lens=[],
61 + subquery_lens=[],
62 + lora_index_mapping=[],
63 + lora_prompt_mapping=[],
64 + lora_requests=set(),
65 + multi_modal_input=None,
66 + slot_mapping=[],
67 + )
68 +
69 +
70 +class PrepareDecodeMetadata(NamedTuple):
71 + input_tokens: List[int]
72 + input_positions: List[int]
73 + attn_metadata: Optional[AttentionMetadata]
74 + lora_index_mapping: List[int]
75 + lora_prompt_mapping: List[int]
76 + lora_requests: Set[LoRARequest]
77 + slot_mapping: List[int]
78 +
79 + @classmethod
80 + def empty(cls):
81 + return PrepareDecodeMetadata(
82 + input_tokens=[],
83 + input_positions=[],
84 + attn_metadata=None,
85 + lora_index_mapping=[],
86 + lora_prompt_mapping=[],
87 + lora_requests=set(),
88 + slot_mapping=[],
89 + )
90 +
91 +
92 +# How batches are constructed.
93 +class BatchType(IntEnum):
94 + # Every batch is prefill.
95 + PREFILL = 0
96 + # Every batch is decode.
97 + DECODE = 1
98 + # Batch is a mixture of prefill and decode.
99 + MIXED = 2
100 +
101 +
102 class ModelRunner:
103
104 def __init__(
@@ -152,10 +214,7 @@ class ModelRunner:
214 def _prepare_prompt(
215 self,
216 seq_group_metadata_list: List[SequenceGroupMetadata],
155 - ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
156 - List[int], List[int], List[int], Set[LoRARequest],
157 - torch.Tensor]:
158 - assert len(seq_group_metadata_list) > 0
217 + ) -> PreparePromptMetadata:
218 input_tokens: List[int] = []
219 input_positions: List[int] = []
220 slot_mapping: List[int] = []
@@ -169,6 +228,9 @@ class ModelRunner:
228 prefix_block_tables: List[List[int]] = []
229 multi_modal_input_list: List[torch.Tensor] = []
230
231 + if len(seq_group_metadata_list) == 0:
232 + return PreparePromptMetadata.empty()
233 +
234 for seq_group_metadata in seq_group_metadata_list:
235 assert seq_group_metadata.is_prompt
236 seq_ids = list(seq_group_metadata.seq_data.keys())
@@ -178,7 +240,8 @@ class ModelRunner:
240 computed_block_nums = seq_group_metadata.computed_block_nums
241 if (self.scheduler_config is not None
242 and self.scheduler_config.chunked_prefill_enabled
181 - and computed_block_nums is not None):
243 + and not (computed_block_nums is None
244 + or computed_block_nums == [])):
245 raise RuntimeError(
246 "chunked prefill cannot be used with prefix caching "
247 "now.")
@@ -190,13 +253,8 @@ class ModelRunner:
253 # it contains output tokens.
254 prefill_end = min(seq_data.get_len(),
255 computed_len + token_chunk_size)
193 - # TODO(sang): Rename it after chunked prefill is introduced.
256 prompt_tokens = seq_data.get_token_ids()[computed_len:prefill_end]
195 - prompt_len = len(prompt_tokens)
196 - # Right now, the prefill_end is always same as the length of
197 - # sequence. However, once chunked prefill is introduced, this
198 - # assumption can be changed.
199 - assert prefill_end == seq_data.get_len()
257 + prompt_len = prefill_end
258 prompt_lens.append(prompt_len)
259
260 # NOTE: This only works for oooooooxxx style attention.
@@ -206,6 +264,14 @@ class ModelRunner:
264 computed_len = len(computed_block_nums) * self.block_size
265 prompt_tokens = prompt_tokens[computed_len:]
266 prefix_block_tables.append(computed_block_nums)
267 + elif self.scheduler_config.chunked_prefill_enabled:
268 + if seq_group_metadata.block_tables is not None:
269 + # Prefill has chunked before.
270 + block_table = seq_group_metadata.block_tables[seq_id]
271 + prefix_block_tables.append(block_table)
272 + else:
273 + # The first prefill.
274 + prefix_block_tables.append([])
275 else:
276 prefix_block_tables.append([])
277 # Right now, prefill start is always 0. However, this
@@ -267,20 +333,8 @@ class ModelRunner:
333
334 max_subquery_len = max(subquery_lens)
335 max_prompt_len = max(prompt_lens)
270 - num_prompt_tokens = len(input_tokens)
336 assert max_subquery_len > 0
337
273 - input_tokens = torch.tensor(input_tokens,
274 - dtype=torch.long,
275 - device=self.device)
276 - input_positions = torch.tensor(input_positions,
277 - dtype=torch.long,
278 - device=self.device)
279 - slot_mapping = torch.tensor(slot_mapping,
280 - dtype=torch.long,
281 - device=self.device)
282 - lora_index_mapping = lora_index_mapping
283 -
338 context_lens_tensor = torch.tensor(context_lens,
339 dtype=torch.int,
340 device=self.device)
@@ -332,11 +386,8 @@ class ModelRunner:
386
387 attn_metadata = self.attn_backend.make_metadata(
388 is_prompt=True,
335 - slot_mapping=slot_mapping,
389 prompt_lens=prompt_lens,
390 prompt_lens_tensor=prompt_lens_tensor,
338 - num_prompt_tokens=num_prompt_tokens,
339 - num_generation_tokens=0,
391 max_subquery_len=max_subquery_len,
392 max_context_len=None,
393 max_prompt_len=max_prompt_len,
@@ -345,18 +396,25 @@ class ModelRunner:
396 context_lens=context_lens_tensor,
397 block_tables=block_tables,
398 use_cuda_graph=False,
348 - kv_cache_dtype=self.kv_cache_dtype,
399 )
350 - return (input_tokens, input_positions, attn_metadata, prompt_lens,
351 - subquery_lens, lora_index_mapping, lora_prompt_mapping,
352 - lora_requests, multi_modal_input)
400 +
401 + return PreparePromptMetadata(
402 + input_tokens=input_tokens,
403 + input_positions=input_positions,
404 + attn_metadata=attn_metadata,
405 + prompt_lens=prompt_lens,
406 + subquery_lens=subquery_lens,
407 + lora_index_mapping=lora_index_mapping,
408 + lora_prompt_mapping=lora_prompt_mapping,
409 + lora_requests=lora_requests,
410 + multi_modal_input=multi_modal_input,
411 + slot_mapping=slot_mapping,
412 + )
413
414 def _prepare_decode(
415 self,
416 seq_group_metadata_list: List[SequenceGroupMetadata],
357 - ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
358 - List[int], Set[LoRARequest]]:
359 - assert len(seq_group_metadata_list) > 0
417 + ) -> PrepareDecodeMetadata:
418 input_tokens: List[int] = []
419 input_positions: List[int] = []
420 slot_mapping: List[int] = []
@@ -366,6 +424,9 @@ class ModelRunner:
424 lora_prompt_mapping: List[int] = []
425 lora_requests: Set[LoRARequest] = set()
426
427 + if len(seq_group_metadata_list) == 0:
428 + return PrepareDecodeMetadata.empty()
429 +
430 for seq_group_metadata in seq_group_metadata_list:
431 assert not seq_group_metadata.is_prompt
432 assert seq_group_metadata.token_chunk_size == 1
@@ -424,15 +485,6 @@ class ModelRunner:
485 lora_index_mapping.append(0)
486 batch_size = graph_batch_size
487
427 - input_tokens = torch.tensor(input_tokens,
428 - dtype=torch.long,
429 - device=self.device)
430 - input_positions = torch.tensor(input_positions,
431 - dtype=torch.long,
432 - device=self.device)
433 - slot_mapping = torch.tensor(slot_mapping,
434 - dtype=torch.long,
435 - device=self.device)
488 context_lens = torch.tensor(context_lens,
489 dtype=torch.int,
490 device=self.device)
@@ -440,9 +492,9 @@ class ModelRunner:
492 if use_captured_graph:
493 # When using cuda-graph all these tensors should be
494 # padded.
443 - assert context_lens.shape[0] == input_tokens.shape[0]
444 - assert context_lens.shape[0] == input_positions.shape[0]
445 - assert context_lens.shape[0] == slot_mapping.shape[0]
495 + assert context_lens.shape[0] == len(input_tokens)
496 + assert context_lens.shape[0] == len(input_positions)
497 + assert context_lens.shape[0] == len(slot_mapping)
498
499 # The shape of graph_block_tables is
500 # [max batch size, max context len // block size].
@@ -464,11 +516,8 @@ class ModelRunner:
516
517 attn_metadata = self.attn_backend.make_metadata(
518 is_prompt=False,
467 - slot_mapping=slot_mapping,
519 prompt_lens=None,
520 prompt_lens_tensor=None,
470 - num_prompt_tokens=0,
471 - num_generation_tokens=len(input_tokens),
521 max_subquery_len=None,
522 max_context_len=max_context_len,
523 max_prompt_len=None,
@@ -477,10 +526,16 @@ class ModelRunner:
526 context_lens=context_lens,
527 block_tables=block_tables,
528 use_cuda_graph=use_captured_graph,
480 - kv_cache_dtype=self.kv_cache_dtype,
529 )
482 - return (input_tokens, input_positions, attn_metadata,
483 - lora_index_mapping, lora_prompt_mapping, lora_requests)
530 + return PrepareDecodeMetadata(
531 + input_tokens=input_tokens,
532 + input_positions=input_positions,
533 + attn_metadata=attn_metadata,
534 + lora_index_mapping=lora_index_mapping,
535 + lora_prompt_mapping=lora_prompt_mapping,
536 + lora_requests=lora_requests,
537 + slot_mapping=slot_mapping,
538 + )
539
540 def _prepare_sample(
541 self,
@@ -586,26 +641,66 @@ class ModelRunner:
641 ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
642 Set[int], LoRAMapping, torch.Tensor]:
643 if self.is_driver_worker:
589 - # NOTE: We assume that all sequences in the group are all prompts or
590 - # all decodes.
591 - is_prompt = seq_group_metadata_list[0].is_prompt
644 + prefill_reqs = []
645 + decode_reqs = []
646 + for seq_group_meta in seq_group_metadata_list:
647 + if seq_group_meta.is_prompt:
648 + prefill_reqs.append(seq_group_meta)
649 + else:
650 + decode_reqs.append(seq_group_meta)
651 +
652 # Prepare input tensors.
593 - if is_prompt:
594 - (input_tokens, input_positions, attn_metadata, prompt_lens,
595 - subquery_lens, lora_index_mapping, lora_prompt_mapping,
596 - lora_requests, multi_modal_input
597 - ) = self._prepare_prompt(seq_group_metadata_list)
598 - else:
599 - (input_tokens, input_positions, attn_metadata,
600 - lora_index_mapping, lora_prompt_mapping,
601 - lora_requests) = self._prepare_decode(seq_group_metadata_list)
602 - prompt_lens = []
603 - subquery_lens = None
604 - multi_modal_input = None
653 + (
654 + input_tokens,
655 + input_positions,
656 + prefill_attn_metadata,
657 + prompt_lens,
658 + subquery_lens,
659 + lora_index_mapping,
660 + lora_prompt_mapping,
661 + lora_requests,
662 + multi_modal_input,
663 + slot_mapping,
664 + ) = self._prepare_prompt(prefill_reqs)
665 + (
666 + decode_input_tokens,
667 + decode_input_positions,
668 + decode_attn_metadata,
669 + decode_lora_index_mapping,
670 + decode_lora_prompt_mapping,
671 + decode_lora_requests,
672 + decode_slot_mapping,
673 + ) = self._prepare_decode(decode_reqs)
674 sampling_metadata = self._prepare_sample(seq_group_metadata_list,
675 prompt_lens,
676 subquery_lens)
677
678 + if not self.scheduler_config.chunked_prefill_enabled:
679 + assert (len(prefill_reqs) and len(decode_reqs)) == 0
680 +
681 + num_prefills = len(prompt_lens)
682 + num_prefill_tokens = len(input_tokens)
683 + num_decode_tokens = len(decode_input_tokens)
684 +
685 + # Coalesce tensors. Note that attn_metadata is currently not
686 + # coalesced for simplicity.
687 + input_tokens.extend(decode_input_tokens)
688 + input_positions.extend(decode_input_positions)
689 + slot_mapping.extend(decode_slot_mapping)
690 + lora_index_mapping.extend(decode_lora_index_mapping)
691 + lora_prompt_mapping.extend(decode_lora_prompt_mapping)
692 + lora_requests.update(decode_lora_requests)
693 +
694 + input_tokens = torch.tensor(input_tokens,
695 + dtype=torch.long,
696 + device=self.device)
697 + input_positions = torch.tensor(input_positions,
698 + dtype=torch.long,
699 + device=self.device)
700 + slot_mapping = torch.tensor(slot_mapping,
701 + dtype=torch.long,
702 + device=self.device)
703 +
704 if self.lora_config:
705 lora_mapping = LoRAMapping(
706 lora_index_mapping,
@@ -615,6 +710,16 @@ class ModelRunner:
710 lora_mapping = None
711
712 # Broadcast the metadata.
713 + # If batch contains both prefill and decode, it sends 2 broadcasts.
714 + # If it only contains 1 type, it triggers a single broadcast.
715 + if (prefill_attn_metadata is not None
716 + and decode_attn_metadata is not None):
717 + batch_type = BatchType.MIXED
718 + elif prefill_attn_metadata is not None:
719 + batch_type = BatchType.PREFILL
720 + else:
721 + batch_type = BatchType.DECODE
722 +
723 metadata_dict = {
724 "input_tokens": input_tokens,
725 "input_positions": input_positions,
@@ -623,19 +728,49 @@ class ModelRunner:
728 "lora_requests": lora_requests,
729 "lora_mapping": lora_mapping,
730 "multi_modal_input": multi_modal_input,
731 + "num_prefill_tokens": num_prefill_tokens,
732 + "num_decode_tokens": num_decode_tokens,
733 + "slot_mapping": slot_mapping,
734 + "num_prefills": num_prefills,
735 + "batch_type": batch_type,
736 }
627 - metadata_dict.update(attn_metadata.asdict_zerocopy())
737 + if prefill_attn_metadata is not None:
738 + metadata_dict.update(prefill_attn_metadata.asdict_zerocopy())
739 + else:
740 + metadata_dict.update(decode_attn_metadata.asdict_zerocopy())
741 broadcast_tensor_dict(metadata_dict, src=0)
742 +
743 + # Broadcast decode attn metadata for mixed batch type.
744 + # The additional broadcast costs 300us overhead on 4 A10 GPUs.
745 + # We can potentially reduce the overhead by coelescing tensors.
746 + if batch_type == BatchType.MIXED:
747 + assert decode_attn_metadata is not None
748 + metadata_dict = decode_attn_metadata.asdict_zerocopy()
749 + broadcast_tensor_dict(metadata_dict, src=0)
750 else:
751 metadata_dict = broadcast_tensor_dict(src=0)
752 input_tokens = metadata_dict.pop("input_tokens")
753 input_positions = metadata_dict.pop("input_positions")
754 + slot_mapping = metadata_dict.pop("slot_mapping")
755 + num_prefills = metadata_dict.pop("num_prefills")
756 selected_token_indices = metadata_dict.pop(
757 "selected_token_indices")
758 lora_mapping = metadata_dict.pop("lora_mapping")
759 lora_requests = metadata_dict.pop("lora_requests")
760 multi_modal_input = metadata_dict.pop("multi_modal_input")
638 - attn_metadata = self.attn_backend.make_metadata(**metadata_dict)
761 + num_prefill_tokens = metadata_dict.pop("num_prefill_tokens")
762 + num_decode_tokens = metadata_dict.pop("num_decode_tokens")
763 + batch_type = metadata_dict.pop("batch_type")
764 +
765 + # Create an attention metadata.
766 + prefill_attn_metadata = None
767 + decode_attn_metadata = None
768 + if batch_type == BatchType.PREFILL or batch_type == BatchType.MIXED:
769 + prefill_attn_metadata = self.attn_backend.make_metadata(
770 + **metadata_dict)
771 + else:
772 + decode_attn_metadata = self.attn_backend.make_metadata(
773 + **metadata_dict)
774 sampling_metadata = SamplingMetadata(
775 seq_groups=None,
776 seq_data=None,
@@ -646,6 +781,23 @@ class ModelRunner:
781 perform_sampling=False,
782 )
783
784 + # if it is a mixed batch, decode attn_metadata is broadcasted
785 + # separately.
786 + if batch_type == BatchType.MIXED:
787 + metadata_dict = broadcast_tensor_dict(src=0)
788 + decode_attn_metadata = self.attn_backend.make_metadata(
789 + **metadata_dict)
790 +
791 + attn_metadata = AttentionMetadata(
792 + num_prefills=num_prefills,
793 + slot_mapping=slot_mapping,
794 + num_prefill_tokens=num_prefill_tokens,
795 + num_decode_tokens=num_decode_tokens,
796 + prefill_metadata=prefill_attn_metadata,
797 + decode_metadata=decode_attn_metadata,
798 + kv_cache_dtype=self.kv_cache_dtype,
799 + )
800 +
801 return (input_tokens, input_positions, attn_metadata,
802 sampling_metadata, lora_requests, lora_mapping,
803 multi_modal_input)
@@ -663,8 +815,10 @@ class ModelRunner:
815 if self.lora_config:
816 self.set_active_loras(lora_requests, lora_mapping)
817
666 - # Execute the model.
667 - if attn_metadata.use_cuda_graph:
818 + # Currently cuda graph is only supported by the decode phase.
819 + prefill_meta = attn_metadata.prefill_metadata
820 + decode_meta = attn_metadata.decode_metadata
821 + if prefill_meta is None and decode_meta.use_cuda_graph:
822 graph_batch_size = input_tokens.shape[0]
823 model_executable = self.graph_runners[graph_batch_size]
824 else:
@@ -842,13 +996,10 @@ class ModelRunner:
996 # memory usage of CUDA graph.
997 for batch_size in reversed(batch_size_capture_list):
998 # Create dummy attn_metadata.
845 - attn_metadata = self.attn_backend.make_metadata(
999 + decode_metadata = self.attn_backend.make_metadata(
1000 is_prompt=False,
847 - slot_mapping=slot_mapping[:batch_size],
1001 prompt_lens=None,
1002 prompt_lens_tensor=None,
850 - num_prompt_tokens=0,
851 - num_generation_tokens=batch_size,
1003 max_subquery_len=None,
1004 max_context_len=self.max_context_len_to_capture,
1005 max_prompt_len=None,
@@ -857,6 +1008,14 @@ class ModelRunner:
1008 context_lens=context_lens[:batch_size],
1009 block_tables=block_tables[:batch_size],
1010 use_cuda_graph=True,
1011 + )
1012 + attn_metadata = AttentionMetadata(
1013 + num_prefills=0,
1014 + num_prefill_tokens=0,
1015 + num_decode_tokens=batch_size,
1016 + slot_mapping=slot_mapping[:batch_size],
1017 + prefill_metadata=None,
1018 + decode_metadata=decode_metadata,
1019 kv_cache_dtype=self.kv_cache_dtype,
1020 )
1021
@@ -950,8 +1109,8 @@ class CUDAGraphRunner:
1109 "positions": positions,
1110 "kv_caches": kv_caches,
1111 "slot_mapping": attn_metadata.slot_mapping,
953 - "context_lens": attn_metadata.context_lens,
954 - "block_tables": attn_metadata.block_tables,
1112 + "context_lens": attn_metadata.decode_metadata.context_lens,
1113 + "block_tables": attn_metadata.decode_metadata.block_tables,
1114 }
1115 self.output_buffers = {"hidden_states": hidden_states}
1116 return
@@ -972,10 +1131,10 @@ class CUDAGraphRunner:
1131 self.input_buffers["positions"].copy_(positions, non_blocking=True)
1132 self.input_buffers["slot_mapping"].copy_(attn_metadata.slot_mapping,
1133 non_blocking=True)
975 - self.input_buffers["context_lens"].copy_(attn_metadata.context_lens,
976 - non_blocking=True)
977 - self.input_buffers["block_tables"].copy_(attn_metadata.block_tables,
978 - non_blocking=True)
1134 + self.input_buffers["context_lens"].copy_(
1135 + attn_metadata.decode_metadata.context_lens, non_blocking=True)
1136 + self.input_buffers["block_tables"].copy_(
1137 + attn_metadata.decode_metadata.block_tables, non_blocking=True)
1138 # Run the graph.
1139 self.graph.replay()
1140