A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core][5/N] Fully working chunked prefill e2e (#3884)
SangBin Cho committed
Apr 11, 2024 at 09:56 UTC
67b4221a61ace91a79aff507df0a95a01978300e
26 files changed
+927
-315
.buildkite/test-pipeline.yaml
+2
@@ -29,6 +29,8 @@ steps:
29
- pytest -v -s test_pynccl.py
30
- TEST_DIST_MODEL=facebook/opt-125m pytest -v -s test_basic_distributed_correctness.py
31
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s test_basic_distributed_correctness.py
32
+ - TEST_DIST_MODEL=facebook/opt-125m pytest -v -s test_chunked_prefill_distributed.py
33
+ - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s test_chunked_prefill_distributed.py
34
35
- label: Engine Test
36
command: pytest -v -s engine tokenization test_sequence.py test_config.py
benchmarks/benchmark_latency.py
+1
-2
@@ -177,8 +177,7 @@ if __name__ == '__main__':
177
help='block size of key/value cache')
178
parser.add_argument(
179
'--enable-chunked-prefill',
180
- type=bool,
181
- default=False,
180
+ action='store_true',
181
help='If True, the prefill requests can be chunked based on the '
182
'max_num_batched_tokens')
183
parser.add_argument(
benchmarks/benchmark_throughput.py
+38
-24
@@ -74,25 +74,31 @@ def run_vllm(
74
quantization_param_path: Optional[str],
75
device: str,
76
enable_prefix_caching: bool,
77
+ enable_chunked_prefill: bool,
78
+ max_num_batched_tokens: int,
79
gpu_memory_utilization: float = 0.9,
80
download_dir: Optional[str] = None,
81
) -> float:
82
from vllm import LLM, SamplingParams
81
- llm = LLM(model=model,
82
- tokenizer=tokenizer,
83
- quantization=quantization,
84
- tensor_parallel_size=tensor_parallel_size,
85
- seed=seed,
86
- trust_remote_code=trust_remote_code,
87
- dtype=dtype,
88
- max_model_len=max_model_len,
89
- gpu_memory_utilization=gpu_memory_utilization,
90
- enforce_eager=enforce_eager,
91
- kv_cache_dtype=kv_cache_dtype,
92
- quantization_param_path=quantization_param_path,
93
- device=device,
94
- enable_prefix_caching=enable_prefix_caching,
95
- download_dir=download_dir)
83
+ llm = LLM(
84
+ model=model,
85
+ tokenizer=tokenizer,
86
+ quantization=quantization,
87
+ tensor_parallel_size=tensor_parallel_size,
88
+ seed=seed,
89
+ trust_remote_code=trust_remote_code,
90
+ dtype=dtype,
91
+ max_model_len=max_model_len,
92
+ gpu_memory_utilization=gpu_memory_utilization,
93
+ enforce_eager=enforce_eager,
94
+ kv_cache_dtype=kv_cache_dtype,
95
+ quantization_param_path=quantization_param_path,
96
+ device=device,
97
+ enable_prefix_caching=enable_prefix_caching,
98
+ download_dir=download_dir,
99
+ enable_chunked_prefill=enable_chunked_prefill,
100
+ max_num_batched_tokens=max_num_batched_tokens,
101
+ )
102
103
# Add the requests to the engine.
104
for prompt, _, output_len in requests:
@@ -213,15 +219,15 @@ def main(args: argparse.Namespace):
219
args.output_len)
220
221
if args.backend == "vllm":
216
- elapsed_time = run_vllm(requests, args.model, args.tokenizer,
217
- args.quantization, args.tensor_parallel_size,
218
- args.seed, args.n, args.use_beam_search,
219
- args.trust_remote_code, args.dtype,
220
- args.max_model_len, args.enforce_eager,
221
- args.kv_cache_dtype,
222
- args.quantization_param_path, args.device,
223
- args.enable_prefix_caching,
224
- args.gpu_memory_utilization, args.download_dir)
222
+ elapsed_time = run_vllm(
223
+ requests, args.model, args.tokenizer, args.quantization,
224
+ args.tensor_parallel_size, args.seed, args.n, args.use_beam_search,
225
+ args.trust_remote_code, args.dtype, args.max_model_len,
226
+ args.enforce_eager, args.kv_cache_dtype,
227
+ args.quantization_param_path, args.device,
228
+ args.enable_prefix_caching, args.enable_chunked_prefill,
229
+ args.max_num_batched_tokens, args.gpu_memory_utilization,
230
+ args.download_dir)
231
elif args.backend == "hf":
232
assert args.tensor_parallel_size == 1
233
elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -335,6 +341,14 @@ if __name__ == "__main__":
341
"--enable-prefix-caching",
342
action='store_true',
343
help="enable automatic prefix caching for vLLM backend.")
344
+ parser.add_argument("--enable-chunked-prefill",
345
+ action='store_true',
346
+ help="enable chunked prefill for vLLM backend.")
347
+ parser.add_argument('--max-num-batched-tokens',
348
+ type=int,
349
+ default=None,
350
+ help='maximum number of batched tokens per '
351
+ 'iteration')
352
parser.add_argument('--download-dir',
353
type=str,
354
default=None,
tests/basic_correctness/test_chunked_prefill.py
new
+70
@@ -0,0 +1,70 @@
1
+"""Compare the outputs of HF and vLLM when using greedy sampling.
2
+
3
+It tests chunked prefill. Chunked prefill can be enabled by
4
+enable_chunked_prefill=True. If prefill size exceeds max_num_batched_tokens,
5
+prefill requests are chunked.
6
+
7
+Run `pytest tests/models/test_chunked_prefill.py`.
8
+"""
9
+import pytest
10
+
11
+MODELS = [
12
+ "facebook/opt-125m",
13
+ "meta-llama/Llama-2-7b-hf",
14
+]
15
+
16
+
17
+@pytest.mark.parametrize("model", MODELS)
18
+@pytest.mark.parametrize("dtype", ["half"])
19
+@pytest.mark.parametrize("max_tokens", [32])
20
+@pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16])
21
+@pytest.mark.parametrize("enforce_eager", [False, True])
22
+# NOTE: Increasing this in this suite will fail CI because we currently cannot
23
+# reset distributed env properly. Use a value > 1 just when you test.
24
+@pytest.mark.parametrize("tensor_parallel_size", [1])
25
+def test_models(
26
+ hf_runner,
27
+ vllm_runner,
28
+ example_prompts,
29
+ model: str,
30
+ dtype: str,
31
+ max_tokens: int,
32
+ chunked_prefill_token_size: int,
33
+ enforce_eager: bool,
34
+ tensor_parallel_size: int,
35
+) -> None:
36
+ if (tensor_parallel_size == 2 and chunked_prefill_token_size != 16
37
+ and not enforce_eager):
38
+ pytest.skip(f"Skip {chunked_prefill_token_size=} and {enforce_eager=} "
39
+ "for high TP to save testing time.")
40
+ max_num_seqs = min(chunked_prefill_token_size, 256)
41
+ enable_chunked_prefill = False
42
+ max_num_batched_tokens = None
43
+ if chunked_prefill_token_size != -1:
44
+ enable_chunked_prefill = True
45
+ max_num_batched_tokens = chunked_prefill_token_size
46
+
47
+ hf_model = hf_runner(model, dtype=dtype)
48
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
49
+ del hf_model
50
+
51
+ vllm_model = vllm_runner(
52
+ model,
53
+ dtype=dtype,
54
+ max_num_batched_tokens=max_num_batched_tokens,
55
+ enable_chunked_prefill=enable_chunked_prefill,
56
+ tensor_parallel_size=tensor_parallel_size,
57
+ enforce_eager=enforce_eager,
58
+ max_num_seqs=max_num_seqs,
59
+ )
60
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
61
+ del vllm_model
62
+ print(vllm_outputs[0])
63
+
64
+ for i in range(len(example_prompts)):
65
+ hf_output_ids, hf_output_str = hf_outputs[i]
66
+ vllm_output_ids, vllm_output_str = vllm_outputs[i]
67
+ assert hf_output_str == vllm_output_str, (
68
+ f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
69
+ assert hf_output_ids == vllm_output_ids, (
70
+ f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
tests/core/test_chunked_prefill_scheduler.py
+8
-8
@@ -104,10 +104,10 @@ def test_chunk():
104
# One chunked prefill, and one decoding.
105
seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
106
assert set(get_sequence_groups(out)) == set(running)
107
- # The first one is decoding.
108
- assert seq_group_meta[0].token_chunk_size == 1
107
+ # The first one is prefill. Scheduler guarantees ordering.
108
+ assert seq_group_meta[0].token_chunk_size == 56
109
# The second one is a chunked prefill.
110
- assert seq_group_meta[1].token_chunk_size == 56
110
+ assert seq_group_meta[1].token_chunk_size == 1
111
assert out.num_prefill_groups == 1
112
assert out.num_batched_tokens == 57
113
@@ -157,12 +157,12 @@ def test_complex():
157
# Decoding & chunked prefill & first chunk of 3rd request is scheduled.
158
seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
159
assert len(get_sequence_groups(out)) == 3
160
- # The first one is decoding.
161
- assert seq_group_meta[0].token_chunk_size == 1
162
- # The second one is a chunked prefill.
160
+ # The first one is the first chunked prefill.
161
+ assert seq_group_meta[0].token_chunk_size == 7
162
+ # The second one is the second new chunked prefill.
163
assert seq_group_meta[1].token_chunk_size == 56
164
- # The third one is also chunked.
165
- assert seq_group_meta[2].token_chunk_size == 7
164
+ # The last one is decode.
165
+ assert seq_group_meta[2].token_chunk_size == 1
166
# Two of them are in chunked prefill.
167
assert out.num_prefill_groups == 2
168
assert out.num_batched_tokens == 64
tests/distributed/test_basic_distributed_correctness.py
+6
-1
@@ -33,11 +33,16 @@ def test_models(
33
dtype: str,
34
max_tokens: int,
35
) -> None:
36
+
37
hf_model = hf_runner(model, dtype=dtype)
38
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
39
del hf_model
40
40
- vllm_model = vllm_runner(model, dtype=dtype, tensor_parallel_size=2)
41
+ vllm_model = vllm_runner(
42
+ model,
43
+ dtype=dtype,
44
+ tensor_parallel_size=2,
45
+ )
46
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
47
del vllm_model
48
tests/distributed/test_chunked_prefill_distributed.py
new
+66
@@ -0,0 +1,66 @@
1
+"""Compare the outputs of HF and distributed vLLM when using greedy sampling.
2
+vLLM will allocate all the available memory, so we need to run the tests one
3
+by one. The solution is to pass arguments (model name) by environment
4
+variables.
5
+
6
+Run:
7
+```sh
8
+TEST_DIST_MODEL=facebook/opt-125m pytest \
9
+ test_chunked_prefill_distributed.py
10
+TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf \
11
+ test_chunked_prefill_distributed.py
12
+```
13
+"""
14
+import os
15
+
16
+import pytest
17
+import torch
18
+
19
+MODELS = [
20
+ os.environ["TEST_DIST_MODEL"],
21
+]
22
+
23
+
24
+@pytest.mark.skipif(torch.cuda.device_count() < 2,
25
+ reason="Need at least 2 GPUs to run the test.")
26
+@pytest.mark.parametrize("model", MODELS)
27
+@pytest.mark.parametrize("dtype", ["half"])
28
+@pytest.mark.parametrize("max_tokens", [5])
29
+@pytest.mark.parametrize("chunked_prefill_token_size", [16])
30
+def test_models(
31
+ hf_runner,
32
+ vllm_runner,
33
+ example_prompts,
34
+ model: str,
35
+ dtype: str,
36
+ max_tokens: int,
37
+ chunked_prefill_token_size: int,
38
+) -> None:
39
+ # Add a chunked prefill config.
40
+ max_num_seqs = min(chunked_prefill_token_size, 256)
41
+ assert chunked_prefill_token_size != -1
42
+ enable_chunked_prefill = True
43
+ max_num_batched_tokens = chunked_prefill_token_size
44
+
45
+ hf_model = hf_runner(model, dtype=dtype)
46
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
47
+ del hf_model
48
+
49
+ vllm_model = vllm_runner(
50
+ model,
51
+ dtype=dtype,
52
+ tensor_parallel_size=2,
53
+ max_num_seqs=max_num_seqs,
54
+ enable_chunked_prefill=enable_chunked_prefill,
55
+ max_num_batched_tokens=max_num_batched_tokens,
56
+ )
57
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
58
+ del vllm_model
59
+
60
+ for i in range(len(example_prompts)):
61
+ hf_output_ids, hf_output_str = hf_outputs[i]
62
+ vllm_output_ids, vllm_output_str = vllm_outputs[i]
63
+ assert hf_output_str == vllm_output_str, (
64
+ f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
65
+ assert hf_output_ids == vllm_output_ids, (
66
+ f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
tests/entrypoints/test_openai_server.py
+1
-1
@@ -141,7 +141,7 @@ def server(zephyr_lora_files):
141
"--max-cpu-loras",
142
"2",
143
"--max-num-seqs",
144
- "128"
144
+ "128",
145
])
146
ray.get(server_runner.ready.remote())
147
yield server_runner
tests/models/test_models.py
+1
-1
@@ -12,7 +12,7 @@ MODELS = [
12
"gpt2",
13
"bigcode/tiny_starcoder_py",
14
"EleutherAI/pythia-70m",
15
- "bigscience/bloom-560m",
15
+ "bigscience/bloom-560m", # Testing alibi slopes.
16
"microsoft/phi-2",
17
"stabilityai/stablelm-3b-4e1t",
18
# "allenai/OLMo-1B", # Broken
tests/worker/test_model_runner.py
+170
-19
@@ -1,14 +1,18 @@
1
import pytest
2
import torch
3
4
-from vllm.config import ModelConfig
4
+from vllm.config import ModelConfig, SchedulerConfig
5
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
6
from vllm.worker.model_runner import ModelRunner, _get_graph_batch_size
7
8
9
@pytest.mark.parametrize("batch_size", list(range(1, 257)))
10
def test_prepare_prompt(batch_size):
11
- model_runner = ModelRunner(None, None, None, None, None)
11
+ scheduler_config = SchedulerConfig(100000,
12
+ 100000,
13
+ 100000,
14
+ enable_chunked_prefill=False)
15
+ model_runner = ModelRunner(None, None, scheduler_config, None, None)
16
model_runner.set_block_size(16)
17
18
prompt_lens = []
@@ -36,8 +40,10 @@ def test_prepare_prompt(batch_size):
40
prompt_len - 1)
41
selected_token_start_idx += prompt_len
42
(input_tokens, input_positions, attn_metadata, return_prompt_lens, _, _, _,
39
- _, _) = (model_runner._prepare_prompt(seq_group_metadata_list))
43
+ _, _,
44
+ slot_mapping) = (model_runner._prepare_prompt(seq_group_metadata_list))
45
assert return_prompt_lens == prompt_lens
46
+ assert len(slot_mapping) == len(input_tokens)
47
48
# Verify input metadata is correct for prompts.
49
device = model_runner.device
@@ -45,8 +51,6 @@ def test_prepare_prompt(batch_size):
51
assert torch.allclose(attn_metadata.prompt_lens_tensor,
52
torch.tensor(prompt_lens, device=device))
53
assert attn_metadata.prompt_lens == prompt_lens
48
- assert attn_metadata.num_prompt_tokens == sum(prompt_lens)
49
- assert attn_metadata.num_generation_tokens == 0
54
assert attn_metadata.max_prompt_len == max(prompt_lens)
55
56
# Test subquery start locs.
@@ -83,23 +87,22 @@ def test_prepare_prompt(batch_size):
87
assert torch.allclose(attn_metadata.block_tables, expected)
88
# Cuda graph should not be used for prerill.
89
assert attn_metadata.use_cuda_graph is False
86
- assert attn_metadata.kv_cache_dtype == "auto"
90
88
- assert input_tokens.shape == (sum(prompt_lens), )
89
- assert input_positions.shape == (sum(prompt_lens), )
91
+ assert len(input_tokens) == sum(prompt_lens)
92
+ assert len(input_positions) == sum(prompt_lens)
93
torch.testing.assert_close(input_tokens, input_positions)
94
95
sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
96
prompt_lens,
97
subquery_lens=prompt_lens)
95
- assert input_tokens.shape == (sum(prompt_lens), )
96
- assert input_positions.shape == (sum(prompt_lens), )
98
+ assert len(input_tokens) == sum(prompt_lens)
99
+ assert len(input_positions) == sum(prompt_lens)
100
actual = sampling_metadata.selected_token_indices
101
expected = torch.tensor(expected_selected_token_indices,
102
device=actual.device,
103
dtype=actual.dtype)
104
torch.testing.assert_close(actual, expected)
102
- torch.testing.assert_close(input_tokens, input_positions)
105
+ assert input_tokens == input_positions
106
107
actual = sampling_metadata.selected_token_indices
108
expected = torch.tensor(expected_selected_token_indices,
@@ -122,7 +125,12 @@ def test_prepare_decode_cuda_graph(batch_size):
125
revision=None,
126
enforce_eager=False,
127
)
125
- model_runner = ModelRunner(model_config, None, None, None, None)
128
+ scheduler_config = SchedulerConfig(100000,
129
+ 100000,
130
+ 100000,
131
+ enable_chunked_prefill=False)
132
+ model_runner = ModelRunner(model_config, None, scheduler_config, None,
133
+ None)
134
model_runner.set_block_size(16)
135
136
prompt_lens = []
@@ -143,16 +151,15 @@ def test_prepare_decode_cuda_graph(batch_size):
151
assert seq_group_metadata.token_chunk_size == 1
152
seq_group_metadata_list.append(seq_group_metadata)
153
146
- input_tokens, input_positions, attn_metadata, _, _, _ = (
154
+ input_tokens, input_positions, attn_metadata, _, _, _, slot_mapping = (
155
model_runner._prepare_decode(seq_group_metadata_list))
156
+ assert len(slot_mapping) == len(input_tokens)
157
158
expected_bs = _get_graph_batch_size(len(seq_group_metadata_list))
159
# Verify input metadata is correct for prompts.
160
device = model_runner.device
161
assert attn_metadata.is_prompt is False
162
assert attn_metadata.prompt_lens is None
154
- assert attn_metadata.num_prompt_tokens == 0
155
- assert attn_metadata.num_generation_tokens == expected_bs
163
assert attn_metadata.max_prompt_len is None
164
assert attn_metadata.subquery_start_loc is None
165
assert attn_metadata.seq_start_loc is None
@@ -170,11 +177,10 @@ def test_prepare_decode_cuda_graph(batch_size):
177
model_runner.get_max_block_per_batch())
178
# Cuda graph should not be used for prerill.
179
assert attn_metadata.use_cuda_graph is True
173
- assert attn_metadata.kv_cache_dtype == "auto"
180
175
- assert input_tokens.shape == (expected_bs, )
176
- assert input_positions.shape == (expected_bs, )
177
- torch.testing.assert_close(input_tokens, input_positions)
181
+ assert len(input_tokens) == expected_bs
182
+ assert len(input_positions) == expected_bs
183
+ assert input_tokens == input_positions
184
185
# Verify Sampling
186
expected_selected_token_indices = []
@@ -190,3 +196,148 @@ def test_prepare_decode_cuda_graph(batch_size):
196
device=actual.device,
197
dtype=actual.dtype)
198
torch.testing.assert_close(actual, expected)
199
+
200
+
201
+def test_empty_seq_group():
202
+ """Verify prepare prompt and decode returns empty output."""
203
+ model_config = ModelConfig(
204
+ "facebook/opt-125m",
205
+ "facebook/opt-125m",
206
+ tokenizer_mode="auto",
207
+ trust_remote_code=False,
208
+ download_dir=None,
209
+ load_format="dummy",
210
+ seed=0,
211
+ dtype="float16",
212
+ revision=None,
213
+ enforce_eager=False,
214
+ )
215
+ model_runner = ModelRunner(model_config, None, None, None, None)
216
+ model_runner.set_block_size(16)
217
+ seq_group_metadata_list = []
218
+ input_tokens, input_positions, attn_metadata, _, _, _, slot_mapping = (
219
+ model_runner._prepare_decode(seq_group_metadata_list))
220
+ assert len(input_tokens) == 0
221
+ assert len(input_positions) == 0
222
+ assert attn_metadata is None
223
+ assert len(slot_mapping) == 0
224
+
225
+ (input_tokens, input_positions, attn_metadata, return_prompt_lens, _, _, _,
226
+ _, _,
227
+ slot_mapping) = (model_runner._prepare_prompt(seq_group_metadata_list))
228
+ assert len(input_tokens) == 0
229
+ assert len(input_positions) == 0
230
+ assert attn_metadata is None
231
+ assert len(slot_mapping) == 0
232
+ assert len(return_prompt_lens) == 0
233
+
234
+
235
+@pytest.mark.parametrize("batch_size", list(range(2, 128)))
236
+@pytest.mark.parametrize("enforce_eager", [True, False])
237
+def test_hybrid_batches(batch_size, enforce_eager, monkeypatch):
238
+
239
+ def get_world_size(group=None):
240
+ return 1
241
+
242
+ def mock_get_process_group_ranks(group=None):
243
+ return [0]
244
+
245
+ monkeypatch.setattr(torch.distributed, "get_world_size", get_world_size)
246
+ monkeypatch.setattr(torch.distributed, "get_process_group_ranks",
247
+ mock_get_process_group_ranks)
248
+
249
+ model_config = ModelConfig(
250
+ "facebook/opt-125m",
251
+ "facebook/opt-125m",
252
+ tokenizer_mode="auto",
253
+ trust_remote_code=False,
254
+ download_dir=None,
255
+ load_format="dummy",
256
+ seed=0,
257
+ dtype="float16",
258
+ revision=None,
259
+ enforce_eager=enforce_eager,
260
+ )
261
+ scheduler_config = SchedulerConfig(100000,
262
+ 100000,
263
+ 100000,
264
+ enable_chunked_prefill=True)
265
+ model_runner = ModelRunner(model_config,
266
+ None,
267
+ scheduler_config,
268
+ None,
269
+ None,
270
+ is_driver_worker=True)
271
+ model_runner.set_block_size(16)
272
+
273
+ # Add prefill requests.
274
+ prompt_lens = []
275
+ seq_group_metadata_list = []
276
+ prefill_metadata_list = []
277
+ decode_metadata_list = []
278
+ block_tables = {0: [1]}
279
+ prefill_batch_size = batch_size // 2
280
+ decode_batch_size = batch_size - prefill_batch_size
281
+ for i in range(prefill_batch_size):
282
+ # make sure all tokens fit into one block
283
+ prompt_len = i % (model_runner.block_size - 1) + 1
284
+ prompt_lens.append(prompt_len)
285
+ seq_data = SequenceData(list(range(prompt_len)))
286
+ seq_group_metadata = SequenceGroupMetadata(
287
+ request_id=f"test_{i}",
288
+ is_prompt=True,
289
+ seq_data={0: seq_data},
290
+ sampling_params=SamplingParams(temperature=0),
291
+ block_tables=block_tables,
292
+ )
293
+ assert seq_group_metadata.token_chunk_size == seq_data.get_len()
294
+ seq_group_metadata_list.append(seq_group_metadata)
295
+ prefill_metadata_list.append(seq_group_metadata)
296
+
297
+ # Add decode requests
298
+ for i in range(prefill_batch_size, batch_size):
299
+ # make sure all tokens fit into one block
300
+ prompt_len = i % (model_runner.block_size - 1) + 1
301
+ prompt_toks = list(range(prompt_len))
302
+ seq_data = SequenceData(prompt_toks)
303
+ seq_group_metadata = SequenceGroupMetadata(
304
+ request_id=f"test_{i}",
305
+ is_prompt=False,
306
+ seq_data={0: seq_data},
307
+ sampling_params=SamplingParams(temperature=0),
308
+ block_tables={0: [1]},
309
+ )
310
+ assert seq_group_metadata.token_chunk_size == 1
311
+ seq_group_metadata_list.append(seq_group_metadata)
312
+ decode_metadata_list.append(seq_group_metadata)
313
+
314
+ (input_tokens, input_positions, attn_metadata, _, _, _,
315
+ _) = model_runner.prepare_input_tensors(seq_group_metadata_list)
316
+
317
+ prefill_meta_actual = attn_metadata.prefill_metadata
318
+ decode_meta_actual = attn_metadata.decode_metadata
319
+
320
+ assert len(attn_metadata.slot_mapping) == len(input_tokens)
321
+ assert len(input_positions) == len(input_tokens)
322
+ assert attn_metadata.kv_cache_dtype == "auto"
323
+ assert attn_metadata.num_prefills == prefill_batch_size
324
+ if enforce_eager:
325
+ assert attn_metadata.num_decode_tokens == decode_batch_size
326
+ else:
327
+ assert attn_metadata.num_decode_tokens == _get_graph_batch_size(
328
+ decode_batch_size)
329
+ assert attn_metadata.num_prefill_tokens == sum(prompt_lens)
330
+
331
+ # Verify attn metadata is consistent. We don't need to test individual
332
+ # values here because they are tested above.
333
+ prefill_meta = model_runner._prepare_prompt(
334
+ prefill_metadata_list).attn_metadata
335
+ decode_meta = model_runner._prepare_decode(
336
+ decode_metadata_list).attn_metadata
337
+
338
+ for attr_expected, attr_actual in zip(vars(prefill_meta),
339
+ vars(prefill_meta_actual)):
340
+ assert attr_expected[1] == attr_actual[1]
341
+ for attr_expected, attr_actual in zip(vars(decode_meta),
342
+ vars(decode_meta_actual)):
343
+ assert attr_expected[1] == attr_actual[1]
vllm/attention/__init__.py
+3
-1
@@ -1,5 +1,6 @@
1
from vllm.attention.backends.abstract import (AttentionBackend,
2
- AttentionMetadata)
2
+ AttentionMetadata,
3
+ AttentionMetadataPerStage)
4
from vllm.attention.layer import Attention
5
from vllm.attention.selector import get_attn_backend
6
@@ -8,4 +9,5 @@ __all__ = [
9
"AttentionMetadata",
10
"Attention",
11
"get_attn_backend",
12
+ "AttentionMetadataPerStage",
13
]
vllm/attention/backends/abstract.py
+39
-3
@@ -1,6 +1,6 @@
1
from abc import ABC, abstractmethod
2
from dataclasses import dataclass, fields
3
-from typing import Any, Dict, List, Optional, Tuple, Type
3
+from typing import Any, Dict, Generic, List, Optional, Tuple, Type, TypeVar
4
5
import torch
6
@@ -47,7 +47,8 @@ class AttentionBackend(ABC):
47
48
49
@dataclass
50
-class AttentionMetadata:
50
+class AttentionMetadataPerStage:
51
+ """Attention metadata for a specific stage. I.e., prefill or decode."""
52
53
def asdict_zerocopy(self) -> Dict[str, Any]:
54
"""Similar to dataclasses.asdict, but avoids deepcopying."""
@@ -59,6 +60,41 @@ class AttentionMetadata:
60
}
61
62
63
+T = TypeVar("T", bound=AttentionMetadataPerStage)
64
+
65
+
66
+@dataclass
67
+class AttentionMetadata(Generic[T]):
68
+ """Attention metadata for prefill and decode batched together."""
69
+ # Total number of prefill requests.
70
+ num_prefills: int
71
+ # Number of prefill tokens.
72
+ num_prefill_tokens: int
73
+ # Number of decode tokens. Note that it is equivalent to the number of
74
+ # decode requests.
75
+ num_decode_tokens: int
76
+ # The attention metadata for prefill requests in a batch.
77
+ # None if there's no prefill requests in a batch.
78
+ prefill_metadata: Optional[T]
79
+ # The attention metadata for decode requests in a batch.
80
+ # None if there's no decode requests in a batch.
81
+ decode_metadata: Optional[T]
82
+ # (num_tokens,). The indices of the token slots that input tokens will be
83
+ # stored into. E.g., if `slot_mapping` is [35, 2, 17] and the block size
84
+ # is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot
85
+ # in block 0, and 1st slot in block 1, respectively.
86
+ slot_mapping: torch.Tensor
87
+ # The kv cache's data type.
88
+ kv_cache_dtype: str
89
+
90
+ def __post_init__(self):
91
+ if self.num_prefill_tokens > 0:
92
+ assert self.num_prefills > 0
93
+ assert self.prefill_metadata is not None
94
+ if self.num_decode_tokens > 0:
95
+ assert self.decode_metadata is not None
96
+
97
+
98
class AttentionImpl(ABC):
99
100
@abstractmethod
@@ -80,7 +116,7 @@ class AttentionImpl(ABC):
116
key: torch.Tensor,
117
value: torch.Tensor,
118
kv_cache: torch.Tensor,
83
- attn_metadata: AttentionMetadata,
119
+ attn_metadata: AttentionMetadata[AttentionMetadataPerStage],
120
kv_scale: float,
121
) -> torch.Tensor:
122
raise NotImplementedError
vllm/attention/backends/flash_attn.py
+55
-30
@@ -11,7 +11,8 @@ import torch
11
from flash_attn import flash_attn_varlen_func
12
13
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14
- AttentionMetadata)
14
+ AttentionMetadata,
15
+ AttentionMetadataPerStage)
16
from vllm.attention.ops.paged_attn import (PagedAttention,
17
PagedAttentionMetadata)
18
@@ -53,7 +54,8 @@ class FlashAttentionBackend(AttentionBackend):
54
55
56
@dataclass
56
-class FlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
57
+class FlashAttentionMetadata(AttentionMetadataPerStage,
58
+ PagedAttentionMetadata):
59
"""Metadata for FlashAttentionBackend.
60
61
NOTE: Any python object stored here is not updated when it is
@@ -68,10 +70,6 @@ class FlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
70
prompt_lens: Optional[List[int]]
71
# prompt_lens stored as a tensor.
72
prompt_lens_tensor: Optional[torch.Tensor]
71
- # The number of prompt tokens. Doesn't include padding.
72
- num_prompt_tokens: int
73
- # The number of generation tokens. Doesn't include padding.
74
- num_generation_tokens: int
73
74
# NOTE(sang): Definition of context_len, subquery_len, and seqlen.
75
# |---------- N-1 iteration --------|
@@ -107,18 +105,27 @@ class FlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
105
class FlashAttentionImpl(AttentionImpl):
106
"""
107
If the input tensors contain prompt tokens, the layout is as follows:
110
- |<--------------- num_prompt_tokens -------------->|
111
- |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->|
108
+ |<--------------- num_prefill_tokens ----------------->|
109
+ |<--prefill_0-->|<--prefill_1-->|...|<--prefill_N-1--->|
110
111
Otherwise, the layout is as follows:
114
- |<------------------ num_generation_tokens (M) ----------------->|
115
- |<--generation_0-->|..........|<--generation_M-1-->|<--padding-->|
112
+ |<----------------- num_decode_tokens ------------------>|
113
+ |<--decode_0-->|..........|<--decode_M-1-->|<--padding-->|
114
115
Generation tokens can contain padding when cuda-graph is used.
116
Currently, prompt tokens don't contain any padding.
117
118
The prompts might have different lengths, while the generation tokens
119
always have length 1.
120
+
121
+ If chunked prefill is enabled, prefill tokens and decode tokens can be
122
+ batched together in a flattened 1D query.
123
+
124
+ |<----- num_prefill_tokens ---->|<------- num_decode_tokens --------->|
125
+ |<-prefill_0->|...|<-prefill_N-1->|<--decode_0-->|...|<--decode_M-1-->|
126
+
127
+ Currently, cuda graph is disabled for chunked prefill, meaning there's no
128
+ padding between prefill and decode tokens.
129
"""
130
131
def __init__(
@@ -155,7 +162,7 @@ class FlashAttentionImpl(AttentionImpl):
162
key: torch.Tensor,
163
value: torch.Tensor,
164
kv_cache: torch.Tensor,
158
- attn_metadata: FlashAttentionMetadata,
165
+ attn_metadata: AttentionMetadata[FlashAttentionMetadata],
166
kv_scale: float,
167
) -> torch.Tensor:
168
"""Forward pass with FlashAttention and PagedAttention.
@@ -188,52 +195,70 @@ class FlashAttentionImpl(AttentionImpl):
195
attn_metadata.kv_cache_dtype,
196
kv_scale)
197
191
- if attn_metadata.is_prompt:
198
+ num_prefill_tokens = attn_metadata.num_prefill_tokens
199
+ num_decode_tokens = attn_metadata.num_decode_tokens
200
+ assert key.shape[0] == num_prefill_tokens + num_decode_tokens
201
+ assert value.shape[0] == num_prefill_tokens + num_decode_tokens
202
+
203
+ output = torch.empty_like(query)
204
+ # Query for decode. KV is not needed because it is already cached.
205
+ decode_query = query[num_prefill_tokens:]
206
+ # QKV for prefill.
207
+ query = query[:num_prefill_tokens]
208
+ key = key[:num_prefill_tokens]
209
+ value = value[:num_prefill_tokens]
210
+
211
+ assert query.shape[0] == num_prefill_tokens
212
+ assert decode_query.shape[0] == num_decode_tokens
213
+
214
+ if prefill_meta := attn_metadata.prefill_metadata:
215
# Prompt run.
193
- if kv_cache is None or attn_metadata.block_tables.numel() == 0:
216
+ if kv_cache is None or prefill_meta.block_tables.numel() == 0:
217
# normal attention
218
# When block_tables are not filled, it means q and k are the
219
# prompt, and they have the same length.
197
- output = flash_attn_varlen_func(
220
+ out = flash_attn_varlen_func(
221
q=query,
222
k=key,
223
v=value,
201
- cu_seqlens_q=attn_metadata.seq_start_loc,
202
- cu_seqlens_k=attn_metadata.seq_start_loc,
203
- max_seqlen_q=attn_metadata.max_prompt_len,
204
- max_seqlen_k=attn_metadata.max_prompt_len,
224
+ cu_seqlens_q=prefill_meta.seq_start_loc,
225
+ cu_seqlens_k=prefill_meta.seq_start_loc,
226
+ max_seqlen_q=prefill_meta.max_prompt_len,
227
+ max_seqlen_k=prefill_meta.max_prompt_len,
228
softmax_scale=self.scale,
229
causal=True,
230
window_size=self.sliding_window,
231
alibi_slopes=self.alibi_slopes,
232
)
233
+ assert output[:num_prefill_tokens].shape == out.shape
234
+ output[:num_prefill_tokens] = out
235
else:
236
# prefix-enabled attention
237
# TODO(Hai) this triton kernel has regression issue (broke) to
238
# deal with different data types between KV and FP8 KV cache,
239
# to be addressed separately.
215
- output = PagedAttention.forward_prefix(
240
+ output[:num_prefill_tokens] = PagedAttention.forward_prefix(
241
query,
242
key,
243
value,
244
key_cache,
245
value_cache,
221
- attn_metadata.block_tables,
222
- attn_metadata.subquery_start_loc,
223
- attn_metadata.prompt_lens_tensor,
224
- attn_metadata.context_lens,
225
- attn_metadata.max_subquery_len,
246
+ prefill_meta.block_tables,
247
+ prefill_meta.subquery_start_loc,
248
+ prefill_meta.prompt_lens_tensor,
249
+ prefill_meta.context_lens,
250
+ prefill_meta.max_subquery_len,
251
self.alibi_slopes,
252
)
228
- else:
253
+ if decode_meta := attn_metadata.decode_metadata:
254
# Decoding run.
230
- output = PagedAttention.forward_decode(
231
- query,
255
+ output[num_prefill_tokens:] = PagedAttention.forward_decode(
256
+ decode_query,
257
key_cache,
258
value_cache,
234
- attn_metadata.block_tables,
235
- attn_metadata.context_lens,
236
- attn_metadata.max_context_len,
259
+ decode_meta.block_tables,
260
+ decode_meta.context_lens,
261
+ decode_meta.max_context_len,
262
attn_metadata.kv_cache_dtype,
263
self.num_kv_heads,
264
self.scale,
vllm/attention/backends/rocm_flash_attn.py
+63
-34
@@ -6,7 +6,8 @@ from typing import Dict, List, Optional, Tuple, Type
6
import torch
7
8
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
9
- AttentionMetadata)
9
+ AttentionMetadata,
10
+ AttentionMetadataPerStage)
11
from vllm.attention.ops.paged_attn import (PagedAttention,
12
PagedAttentionMetadata)
13
from vllm.logger import init_logger
@@ -51,7 +52,8 @@ class ROCmFlashAttentionBackend(AttentionBackend):
52
53
54
@dataclass
54
-class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
55
+class ROCmFlashAttentionMetadata(AttentionMetadataPerStage,
56
+ PagedAttentionMetadata):
57
"""Metadata for FlashAttentionBackend.
58
59
NOTE: Any python object stored here is not updated when it is
@@ -66,10 +68,6 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
68
prompt_lens: Optional[List[int]]
69
# prompt_lens stored as a tensor.
70
prompt_lens_tensor: Optional[torch.Tensor]
69
- # The number of prompt tokens. Doesn't include padding.
70
- num_prompt_tokens: int
71
- # The number of generation tokens. Doesn't include padding.
72
- num_generation_tokens: int
71
72
# NOTE(sang): Definition of context_len, subquery_len, and seqlen.
73
# |---------- N-1 iteration --------|
@@ -117,6 +115,15 @@ class ROCmFlashAttentionImpl(AttentionImpl):
115
116
The prompts might have different lengths, while the generation tokens
117
always have length 1.
118
+
119
+ If chunked prefill is enabled, prefill tokens and decode tokens can be
120
+ batched together in a flattened 1D query.
121
+
122
+ |<----- num_prefill_tokens ---->|<------- num_decode_tokens ----------->|
123
+ |<-prompt_0->|...|<-prompt_N-1->|<-generation_0->|...|<-generation_M-1->|
124
+
125
+ Currently, cuda graph is disabled for chunked prefill, meaning there's no
126
+ padding between prefill and decode tokens.
127
"""
128
129
def __init__(
@@ -181,7 +188,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
188
key: torch.Tensor,
189
value: torch.Tensor,
190
kv_cache: torch.Tensor,
184
- attn_metadata: ROCmFlashAttentionMetadata,
191
+ attn_metadata: AttentionMetadata[ROCmFlashAttentionMetadata],
192
kv_scale: float = 1.0,
193
) -> torch.Tensor:
194
"""Forward pass with FlashAttention and PagedAttention.
@@ -218,9 +225,25 @@ class ROCmFlashAttentionImpl(AttentionImpl):
225
kv_scale,
226
)
227
221
- if attn_metadata.is_prompt:
228
+ num_prefill_tokens = attn_metadata.num_prefill_tokens
229
+ num_decode_tokens = attn_metadata.num_decode_tokens
230
+ assert key.shape[0] == num_prefill_tokens + num_decode_tokens
231
+ assert value.shape[0] == num_prefill_tokens + num_decode_tokens
232
+
233
+ output = torch.empty_like(query)
234
+ # Query for decode. KV is not needed because it is already cached.
235
+ decode_query = query[num_prefill_tokens:]
236
+ # QKV for prefill.
237
+ query = query[:num_prefill_tokens]
238
+ key = key[:num_prefill_tokens]
239
+ value = value[:num_prefill_tokens]
240
+
241
+ assert query.shape[0] == num_prefill_tokens
242
+ assert decode_query.shape[0] == num_decode_tokens
243
+
244
+ if prefill_meta := attn_metadata.prefill_metadata:
245
# Prompt run.
223
- if kv_cache is None or attn_metadata.block_tables.numel() == 0:
246
+ if kv_cache is None or prefill_meta.block_tables.numel() == 0:
247
# triton attention
248
# When block_tables are not filled, it means q and k are the
249
# prompt, and they have the same length.
@@ -230,63 +253,69 @@ class ROCmFlashAttentionImpl(AttentionImpl):
253
key = self.repeat_kv(key, self.num_queries_per_kv)
254
value = self.repeat_kv(value, self.num_queries_per_kv)
255
if self.use_naive_attn:
233
- output = self.attn_fuc(
256
+ out = self.attn_fuc(
257
query,
258
key,
259
value,
237
- attn_metadata.prompt_lens,
260
+ prefill_meta.prompt_lens,
261
self.scale,
262
)
263
+ assert output[:num_prefill_tokens].shape == out.shape
264
+ output[:num_prefill_tokens] = out
265
else:
241
- output, _ = self.attn_func(
266
+ out, _ = self.attn_func(
267
query,
268
key,
269
value,
270
None,
246
- attn_metadata.seq_start_loc,
247
- attn_metadata.seq_start_loc,
248
- attn_metadata.max_prompt_len,
249
- attn_metadata.max_prompt_len,
271
+ prefill_meta.seq_start_loc,
272
+ prefill_meta.seq_start_loc,
273
+ prefill_meta.max_prompt_len,
274
+ prefill_meta.max_prompt_len,
275
True,
276
self.scale,
277
)
278
+ assert output[:num_prefill_tokens].shape == out.shape
279
+ output[:num_prefill_tokens] = out
280
else:
254
- output = self.attn_func(
281
+ out = self.attn_func(
282
q=query,
283
k=key,
284
v=value,
258
- cu_seqlens_q=attn_metadata.seq_start_loc,
259
- cu_seqlens_k=attn_metadata.seq_start_loc,
260
- max_seqlen_q=attn_metadata.max_prompt_len,
261
- max_seqlen_k=attn_metadata.max_prompt_len,
285
+ cu_seqlens_q=prefill_meta.seq_start_loc,
286
+ cu_seqlens_k=prefill_meta.seq_start_loc,
287
+ max_seqlen_q=prefill_meta.max_prompt_len,
288
+ max_seqlen_k=prefill_meta.max_prompt_len,
289
softmax_scale=self.scale,
290
causal=True,
291
)
265
-
292
+ assert output[:num_prefill_tokens].shape == out.shape
293
+ output[:num_prefill_tokens] = out
294
else:
295
# prefix-enabled attention
268
- output = PagedAttention.forward_prefix(
296
+ output[:num_prefill_tokens] = PagedAttention.forward_prefix(
297
query,
298
key,
299
value,
300
key_cache,
301
value_cache,
274
- attn_metadata.block_tables,
275
- attn_metadata.subquery_start_loc,
276
- attn_metadata.prompt_lens_tensor,
277
- attn_metadata.context_lens,
278
- attn_metadata.max_subquery_len,
302
+ prefill_meta.block_tables,
303
+ prefill_meta.subquery_start_loc,
304
+ prefill_meta.prompt_lens_tensor,
305
+ prefill_meta.context_lens,
306
+ prefill_meta.max_subquery_len,
307
self.alibi_slopes,
308
)
281
- else:
309
+
310
+ if decode_meta := attn_metadata.decode_metadata:
311
# Decoding run.
283
- output = PagedAttention.forward_decode(
284
- query,
312
+ output[num_prefill_tokens:] = PagedAttention.forward_decode(
313
+ decode_query,
314
key_cache,
315
value_cache,
287
- attn_metadata.block_tables,
288
- attn_metadata.context_lens,
289
- attn_metadata.max_context_len,
316
+ decode_meta.block_tables,
317
+ decode_meta.context_lens,
318
+ decode_meta.max_context_len,
319
attn_metadata.kv_cache_dtype,
320
self.num_kv_heads,
321
self.scale,
vllm/attention/backends/torch_sdpa.py
+42
-25
@@ -7,7 +7,8 @@ import torch
7
from torch.nn.functional import scaled_dot_product_attention
8
9
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10
- AttentionMetadata)
10
+ AttentionMetadata,
11
+ AttentionMetadataPerStage)
12
from vllm.attention.ops.paged_attn import (PagedAttention,
13
PagedAttentionMetadata)
14
@@ -49,17 +50,14 @@ class TorchSDPABackend(AttentionBackend):
50
51
52
@dataclass
52
-class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
53
+class TorchSDPAMetadata(AttentionMetadataPerStage, PagedAttentionMetadata):
54
"""Metadata for TorchSDPABackend.
55
"""
56
# Currently, input sequences can only contain all prompts
57
# or all decoding. True if all sequences are prompts.
58
is_prompt: bool
58
- slot_mapping: torch.Tensor
59
prompt_lens: Optional[List[int]]
60
prompt_lens_tensor: Optional[torch.Tensor]
61
- num_prompt_tokens: int
62
- num_generation_tokens: int
61
62
max_subquery_len: Optional[int] = None
63
max_prompt_len: Optional[int] = None
@@ -113,7 +111,7 @@ class TorchSDPABackendImpl(AttentionImpl):
111
key: torch.Tensor,
112
value: torch.Tensor,
113
kv_cache: Optional[torch.Tensor],
116
- attn_metadata: TorchSDPAMetadata,
114
+ attn_metadata: AttentionMetadata[TorchSDPAMetadata],
115
kv_scale: float,
116
) -> torch.Tensor:
117
"""Forward pass with torch SDPA and PagedAttention.
@@ -142,36 +140,51 @@ class TorchSDPABackendImpl(AttentionImpl):
140
attn_metadata.kv_cache_dtype,
141
kv_scale)
142
145
- if attn_metadata.is_prompt:
146
- if (kv_cache is None or attn_metadata.block_tables.numel() == 0):
143
+ num_prefill_tokens = attn_metadata.num_prefill_tokens
144
+ num_decode_tokens = attn_metadata.num_decode_tokens
145
+ assert key.shape[0] == num_prefill_tokens + num_decode_tokens
146
+ assert value.shape[0] == num_prefill_tokens + num_decode_tokens
147
+
148
+ output = torch.empty_like(query)
149
+ # Query for decode. KV is not needed because it is already cached.
150
+ decode_query = query[num_prefill_tokens:]
151
+ # QKV for prefill.
152
+ query = query[:num_prefill_tokens]
153
+ key = key[:num_prefill_tokens]
154
+ value = value[:num_prefill_tokens]
155
+
156
+ assert query.shape[0] == num_prefill_tokens
157
+ assert decode_query.shape[0] == num_decode_tokens
158
+
159
+ if prefill_meta := attn_metadata.prefill_metadata:
160
+ if (kv_cache is None or prefill_meta.block_tables.numel() == 0):
161
if self.num_kv_heads != self.num_heads:
162
key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
163
value = value.repeat_interleave(self.num_queries_per_kv,
164
dim=1)
165
152
- if attn_metadata.attn_bias is None:
166
+ if prefill_meta.attn_bias is None:
167
if self.alibi_slopes is not None:
168
att_masks = _make_alibi_bias(
169
self.alibi_slopes, query.dtype,
156
- attn_metadata.prompt_lens) # type: ignore
170
+ prefill_meta.prompt_lens) # type: ignore
171
elif self.sliding_window is not None:
172
att_masks = _make_sliding_window_bias(
159
- attn_metadata.prompt_lens, self.sliding_window,
173
+ prefill_meta.prompt_lens, self.sliding_window,
174
query.dtype) # type: ignore
175
else:
162
- att_masks = [None] * len(attn_metadata.prompt_lens)
163
- attn_metadata.attn_bias = att_masks
176
+ att_masks = [None] * len(prefill_meta.prompt_lens)
177
+ prefill_meta.attn_bias = att_masks
178
179
query = query.movedim(0, query.dim() - 2)
180
key = key.movedim(0, key.dim() - 2)
181
value = value.movedim(0, value.dim() - 2)
182
183
start = 0
170
- output = torch.empty(
171
- (num_tokens, self.num_heads, self.head_size),
172
- dtype=query.dtype)
173
- for prompt_len, mask in zip(attn_metadata.prompt_lens,
174
- attn_metadata.attn_bias):
184
+ out = torch.empty((num_tokens, self.num_heads, self.head_size),
185
+ dtype=query.dtype)
186
+ for prompt_len, mask in zip(prefill_meta.prompt_lens,
187
+ prefill_meta.attn_bias):
188
end = start + prompt_len
189
sub_out = scaled_dot_product_attention(
190
query[:, start:end, :],
@@ -181,28 +194,32 @@ class TorchSDPABackendImpl(AttentionImpl):
194
dropout_p=0.0,
195
is_causal=not self.need_mask,
196
scale=self.scale).movedim(query.dim() - 2, 0)
184
- output[start:end, :, :] = sub_out
197
+ out[start:end, :, :] = sub_out
198
start = end
199
+ assert out.shape == output[:num_prefill_tokens].shape
200
+ output[:num_prefill_tokens] = out
201
else:
202
# prefix-enabled attention
203
raise RuntimeError(
204
"Torch SDPA backend doesn't support prefix decoding.")
205
191
- else:
206
+ if decode_meta := attn_metadata.decode_metadata:
207
# Decoding run.
193
- output = PagedAttention.forward_decode(
194
- query,
208
+ out = PagedAttention.forward_decode(
209
+ decode_query,
210
key_cache,
211
value_cache,
197
- attn_metadata.block_tables,
198
- attn_metadata.context_lens,
199
- attn_metadata.max_context_len,
212
+ decode_meta.block_tables,
213
+ decode_meta.context_lens,
214
+ decode_meta.max_context_len,
215
attn_metadata.kv_cache_dtype,
216
self.num_kv_heads,
217
self.scale,
218
self.alibi_slopes,
219
kv_scale,
220
)
221
+ assert out.shape == output[num_prefill_tokens:].shape
222
+ output[num_prefill_tokens:]
223
224
# Reshape the output tensor.
225
return output.view(-1, self.num_heads * self.head_size)
vllm/attention/backends/xformers.py
+79
-59
@@ -9,7 +9,8 @@ from xformers.ops.fmha.attn_bias import (AttentionBias,
9
LowerTriangularMaskWithTensorBias)
10
11
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
12
- AttentionMetadata)
12
+ AttentionMetadata,
13
+ AttentionMetadataPerStage)
14
from vllm.attention.ops.paged_attn import (PagedAttention,
15
PagedAttentionMetadata)
16
from vllm.logger import init_logger
@@ -54,7 +55,7 @@ class XFormersBackend(AttentionBackend):
55
56
57
@dataclass
57
-class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
58
+class XFormersMetadata(AttentionMetadataPerStage, PagedAttentionMetadata):
59
"""Metadata for XFormersbackend.
60
61
NOTE: Any python object stored here is not updated when it is
@@ -65,19 +66,10 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
66
# Currently, input sequences can only contain all prompts
67
# or all decoding. True if all sequences are prompts.
68
is_prompt: bool
68
- # (num_tokens,). The indices of the token slots that input tokens will be
69
- # stored into. E.g., if `slot_mapping` is [35, 2, 17] and the block size
70
- # is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot
71
- # in block 0, and 1st slot in block 1, respectively.
72
- slot_mapping: torch.Tensor
69
# (batch_size,). The prompt length per sequence. None if it is a decoding.
70
prompt_lens: Optional[List[int]]
71
# prompt_lens stored as a tensor.
72
prompt_lens_tensor: Optional[torch.Tensor]
77
- # The number of prompt tokens. Doesn't include padding.
78
- num_prompt_tokens: int
79
- # The number of generation tokens. Doesn't include padding.
80
- num_generation_tokens: int
73
74
# NOTE(sang): Definition of context_len, subquery_len, and seqlen.
75
# |---------- N-1 iteration --------|
@@ -123,18 +115,27 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
115
class XFormersImpl(AttentionImpl):
116
"""
117
If the input tensors contain prompt tokens, the layout is as follows:
126
- |<--------------- num_prompt_tokens --------------->|
127
- |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1--->|
118
+ |<--------------- num_prefill_tokens ----------------->|
119
+ |<--prefill_0-->|<--prefill_1-->|...|<--prefill_N-1--->|
120
121
Otherwise, the layout is as follows:
130
- |<------------------ num_generation_tokens (M) ----------------->|
131
- |<--generation_0-->|..........|<--generation_M-1-->|<--padding-->|
122
+ |<----------------- num_decode_tokens ------------------>|
123
+ |<--decode_0-->|..........|<--decode_M-1-->|<--padding-->|
124
125
Generation tokens can contain padding when cuda-graph is used.
126
Currently, prompt tokens don't contain any padding.
127
128
The prompts might have different lengths, while the generation tokens
129
always have length 1.
130
+
131
+ If chunked prefill is enabled, prefill tokens and decode tokens can be
132
+ batched together in a flattened 1D query.
133
+
134
+ |<----- num_prefill_tokens ---->|<------- num_decode_tokens --------->|
135
+ |<-prefill_0->|...|<-prefill_N-1->|<--decode_0-->|...|<--decode_M-1-->|
136
+
137
+ Currently, cuda graph is disabled for chunked prefill, meaning there's no
138
+ padding between prefill and decode tokens.
139
"""
140
141
def __init__(
@@ -170,7 +171,7 @@ class XFormersImpl(AttentionImpl):
171
key: torch.Tensor,
172
value: torch.Tensor,
173
kv_cache: Optional[torch.Tensor],
173
- attn_metadata: XFormersMetadata,
174
+ attn_metadata: AttentionMetadata[XFormersMetadata],
175
kv_scale: float,
176
) -> torch.Tensor:
177
"""Forward pass with xFormers and PagedAttention.
@@ -202,59 +203,61 @@ class XFormersImpl(AttentionImpl):
203
attn_metadata.kv_cache_dtype,
204
kv_scale)
205
205
- if attn_metadata.is_prompt:
206
+ num_prefill_tokens = attn_metadata.num_prefill_tokens
207
+ num_decode_tokens = attn_metadata.num_decode_tokens
208
+ assert key.shape[0] == num_prefill_tokens + num_decode_tokens
209
+ assert value.shape[0] == num_prefill_tokens + num_decode_tokens
210
+
211
+ output = torch.empty_like(query)
212
+ # Query for decode. KV is not needed because it is already cached.
213
+ decode_query = query[num_prefill_tokens:]
214
+ # QKV for prefill.
215
+ query = query[:num_prefill_tokens]
216
+ key = key[:num_prefill_tokens]
217
+ value = value[:num_prefill_tokens]
218
+
219
+ assert query.shape[0] == num_prefill_tokens
220
+ assert decode_query.shape[0] == num_decode_tokens
221
+
222
+ if prefill_meta := attn_metadata.prefill_metadata:
223
# Prompt run.
207
- if kv_cache is None or attn_metadata.block_tables.numel() == 0:
224
+ if kv_cache is None or prefill_meta.block_tables.numel() == 0:
225
# normal attention.
226
# block tables are empty if the prompt does not have a cached
227
# prefix.
211
- if self.num_kv_heads != self.num_heads:
212
- # As of Nov 2023, xformers only supports MHA. For MQA/GQA,
213
- # project the key and value tensors to the desired number of
214
- # heads.
215
- # TODO(woosuk): Use MQA/GQA kernels for higher performance.
216
- query = query.view(query.shape[0], self.num_kv_heads,
217
- self.num_queries_per_kv,
218
- query.shape[-1])
219
- key = key[:, :,
220
- None, :].expand(key.shape[0], self.num_kv_heads,
221
- self.num_queries_per_kv,
222
- key.shape[-1])
223
- value = value[:, :,
224
- None, :].expand(value.shape[0],
225
- self.num_kv_heads,
226
- self.num_queries_per_kv,
227
- value.shape[-1])
228
-
229
- output = self._run_memory_efficient_xformers_forward(
230
- query, key, value, attn_metadata)
228
+ out = self._run_memory_efficient_xformers_forward(
229
+ query, key, value, prefill_meta)
230
+ assert out.shape == output[:num_prefill_tokens].shape
231
+ output[:num_prefill_tokens] = out
232
else:
233
# prefix-enabled attention
234
# TODO(Hai) this triton kernel has regression issue (broke) to
235
# deal with different data types between KV and FP8 KV cache,
236
# to be addressed separately.
236
- output = PagedAttention.forward_prefix(
237
+ out = PagedAttention.forward_prefix(
238
query,
239
key,
240
value,
241
key_cache,
242
value_cache,
242
- attn_metadata.block_tables,
243
- attn_metadata.subquery_start_loc,
244
- attn_metadata.prompt_lens_tensor,
245
- attn_metadata.context_lens,
246
- attn_metadata.max_subquery_len,
243
+ prefill_meta.block_tables,
244
+ prefill_meta.subquery_start_loc,
245
+ prefill_meta.prompt_lens_tensor,
246
+ prefill_meta.context_lens,
247
+ prefill_meta.max_subquery_len,
248
self.alibi_slopes,
249
)
249
- else:
250
- # Decoding run.
251
- output = PagedAttention.forward_decode(
252
- query,
250
+ assert output[:num_prefill_tokens].shape == out.shape
251
+ output[:num_prefill_tokens] = out
252
+
253
+ if decode_meta := attn_metadata.decode_metadata:
254
+ output[num_prefill_tokens:] = PagedAttention.forward_decode(
255
+ decode_query,
256
key_cache,
257
value_cache,
255
- attn_metadata.block_tables,
256
- attn_metadata.context_lens,
257
- attn_metadata.max_context_len,
258
+ decode_meta.block_tables,
259
+ decode_meta.context_lens,
260
+ decode_meta.max_context_len,
261
attn_metadata.kv_cache_dtype,
262
self.num_kv_heads,
263
self.scale,
@@ -275,13 +278,30 @@ class XFormersImpl(AttentionImpl):
278
"""Attention for 1D query of multiple prompts. Multiple prompt
279
tokens are flattened in to `query` input.
280
281
+ See https://facebookresearch.github.io/xformers/components/ops.html
282
+ for API spec.
283
+
284
Args:
279
- output: shape = [num_prompt_tokens, num_heads, head_size]
280
- query: shape = [num_prompt_tokens, num_heads, head_size]
281
- key: shape = [num_prompt_tokens, num_kv_heads, head_size]
282
- value: shape = [num_prompt_tokens, num_kv_heads, head_size]
285
+ output: shape = [num_prefill_tokens, num_heads, head_size]
286
+ query: shape = [num_prefill_tokens, num_heads, head_size]
287
+ key: shape = [num_prefill_tokens, num_kv_heads, head_size]
288
+ value: shape = [num_prefill_tokens, num_kv_heads, head_size]
289
attn_metadata: Metadata for attention.
290
"""
291
+ original_query = query
292
+ if self.num_kv_heads != self.num_heads:
293
+ # GQA/MQA requires the shape [B, M, G, H, K].
294
+ # Note that the output also has the same shape (which is different
295
+ # from a spec from the doc).
296
+ query = query.view(query.shape[0], self.num_kv_heads,
297
+ self.num_queries_per_kv, query.shape[-1])
298
+ key = key[:, :,
299
+ None, :].expand(key.shape[0], self.num_kv_heads,
300
+ self.num_queries_per_kv, key.shape[-1])
301
+ value = value[:, :,
302
+ None, :].expand(value.shape[0], self.num_kv_heads,
303
+ self.num_queries_per_kv,
304
+ value.shape[-1])
305
# Set attention bias if not provided. This typically happens at
306
# the very attention layer of every iteration.
307
# FIXME(woosuk): This is a hack.
@@ -302,6 +322,7 @@ class XFormersImpl(AttentionImpl):
322
# TODO(woosuk): Too many view operations. Let's try to reduce
323
# them in the future for code readability.
324
if self.alibi_slopes is None:
325
+ # Add the batch dimension.
326
query = query.unsqueeze(0)
327
key = key.unsqueeze(0)
328
value = value.unsqueeze(0)
@@ -312,14 +333,13 @@ class XFormersImpl(AttentionImpl):
333
attn_bias=attn_metadata.attn_bias[0],
334
p=0.0,
335
scale=self.scale)
315
-
316
- return out.view_as(query)
336
+ return out.view_as(original_query)
337
338
# Attention with alibi slopes.
339
# FIXME(woosuk): Because xformers does not support dynamic sequence
340
# lengths with custom attention bias, we process each prompt one by
341
# one. This is inefficient, especially when we have many short prompts.
322
- output = torch.empty_like(query)
342
+ output = torch.empty_like(original_query)
343
start = 0
344
for i, prompt_len in enumerate(attn_metadata.prompt_lens):
345
end = start + prompt_len
@@ -331,7 +351,7 @@ class XFormersImpl(AttentionImpl):
351
p=0.0,
352
scale=self.scale)
353
# TODO(woosuk): Unnecessary copy. Optimize.
334
- output[start:end].copy_(out.squeeze(0))
354
+ output[start:end].copy_(out.view_as(original_query[start:end]))
355
start += prompt_len
356
return output
357
vllm/attention/layer.py
+3
-2
@@ -4,7 +4,8 @@ from typing import List, Optional
4
import torch
5
import torch.nn as nn
6
7
-from vllm.attention.backends.abstract import AttentionMetadata
7
+from vllm.attention.backends.abstract import (AttentionMetadata,
8
+ AttentionMetadataPerStage)
9
from vllm.attention.selector import get_attn_backend
10
11
@@ -41,7 +42,7 @@ class Attention(nn.Module):
42
key: torch.Tensor,
43
value: torch.Tensor,
44
kv_cache: Optional[torch.Tensor],
44
- attn_metadata: AttentionMetadata,
45
+ attn_metadata: AttentionMetadata[AttentionMetadataPerStage],
46
kv_scale: float = 1.0,
47
) -> torch.Tensor:
48
return self.impl.forward(query, key, value, kv_cache, attn_metadata,
vllm/attention/ops/paged_attn.py
-6
@@ -13,11 +13,6 @@ _PARTITION_SIZE = 512
13
@dataclass
14
class PagedAttentionMetadata:
15
"""Metadata for PagedAttention."""
16
- # (num_tokens,). The indices of the token slots that input tokens will be
17
- # stored into. E.g., if `slot_mapping` is [35, 2, 17] and the block size
18
- # is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot
19
- # in block 0, and 1st slot in block 1, respectively.
20
- slot_mapping: torch.Tensor
16
# (batch_size,). The length of context (tokens stored in KV cache) per
17
# sequence. WARNING: When it is a prefill request, it doesn't include new
18
# tokens. When it is for decoding, it includes a new token.
@@ -31,7 +26,6 @@ class PagedAttentionMetadata:
26
# 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph
27
# captured.
28
block_tables: Optional[torch.Tensor]
34
- kv_cache_dtype: str
29
30
31
class PagedAttention:
vllm/config.py
+10
-3
@@ -565,9 +565,16 @@ class SchedulerConfig:
565
if max_num_batched_tokens is not None:
566
self.max_num_batched_tokens = max_num_batched_tokens
567
else:
568
- # If max_model_len is too short, use 2048 as the default value for
569
- # higher throughput.
570
- self.max_num_batched_tokens = max(max_model_len, 2048)
568
+ if enable_chunked_prefill:
569
+ # For chunked prefill, choose the well-tuned batch size.
570
+ self.max_num_batched_tokens = 768
571
+ else:
572
+ # If max_model_len is too short, use 2048 as the default value
573
+ # for higher throughput.
574
+ self.max_num_batched_tokens = max(max_model_len, 2048)
575
+ if enable_chunked_prefill:
576
+ logger.info("Chunked prefill is enabled (EXPERIMENTAL).")
577
+
578
self.max_num_seqs = max_num_seqs
579
self.max_model_len = max_model_len
580
self.use_v2_block_manager = use_v2_block_manager
vllm/core/scheduler.py
+9
-6
@@ -140,7 +140,11 @@ class SchedulerOutputs:
140
141
@property
142
def lora_requests(self) -> Set[LoRARequest]:
143
- return {g.seq_group.lora_request for g in self.scheduled_seq_groups}
143
+ return {
144
+ g.seq_group.lora_request
145
+ for g in self.scheduled_seq_groups
146
+ if g.seq_group.lora_request is not None
147
+ }
148
149
150
@dataclass
@@ -826,13 +830,12 @@ class Scheduler:
830
# Update swapped requests.
831
self.swapped = remaining_swapped
832
self.swapped.extend(running_scheduled.swapped_out)
829
-
833
return SchedulerOutputs(
834
scheduled_seq_groups=(prefills.seq_groups +
832
- running_scheduled.decode_seq_groups +
835
running_scheduled.prefill_seq_groups +
834
- swapped_in.decode_seq_groups +
835
- swapped_in.prefill_seq_groups),
836
+ swapped_in.prefill_seq_groups +
837
+ running_scheduled.decode_seq_groups +
838
+ swapped_in.decode_seq_groups),
839
num_prefill_groups=(len(prefills.seq_groups) +
840
len(swapped_in.prefill_seq_groups) +
841
len(running_scheduled.prefill_seq_groups)),
@@ -907,7 +910,7 @@ class Scheduler:
910
911
# It assumes the scheduled_seq_groups is ordered by
912
# prefill < decoding.
910
- is_prompt = i < scheduler_outputs.num_prefill_groups
913
+ is_prompt = seq_group.is_prefill()
914
seq_group_metadata = SequenceGroupMetadata(
915
request_id=seq_group.request_id,
916
is_prompt=is_prompt,
vllm/distributed/communication_op.py
+9
-1
@@ -173,10 +173,18 @@ def broadcast_tensor_dict(
173
torch.distributed.broadcast_object_list([metadata_list],
174
src=src,
175
group=group)
176
+ async_handles = []
177
for key, value in metadata_list:
178
if isinstance(value, TensorMetadata):
179
tensor = tensor_dict[key]
179
- torch.distributed.broadcast(tensor, src=src, group=group)
180
+ async_handles.append(
181
+ torch.distributed.broadcast(tensor,
182
+ src=src,
183
+ group=group,
184
+ async_op=True))
185
+ for async_handle in async_handles:
186
+ async_handle.wait()
187
+
188
else:
189
recv_metadata_list = [None]
190
torch.distributed.broadcast_object_list(recv_metadata_list,
vllm/engine/arg_utils.py
+2
-3
@@ -386,9 +386,8 @@ class EngineArgs:
386
'prompt latency) before scheduling next prompt.')
387
parser.add_argument(
388
'--enable-chunked-prefill',
389
- type=bool,
390
- default=False,
391
- help='If True, the prefill requests can be chunked based on the '
389
+ action='store_true',
390
+ help='If set, the prefill requests can be chunked based on the '
391
'max_num_batched_tokens')
392
393
parser.add_argument(
vllm/engine/llm_engine.py
+4
-1
@@ -633,7 +633,10 @@ class LLMEngine:
633
seq_group = scheduled_seq_group.seq_group
634
seq_group.update_num_computed_tokens(
635
scheduled_seq_group.token_chunk_size)
636
- self._process_sequence_group_outputs(seq_group, outputs)
636
+ # If uncomputed tokens > 0, it means prefill is chunked.
637
+ # We don't need to process outputs in that case.
638
+ if seq_group.get_num_uncomputed_tokens() == 0:
639
+ self._process_sequence_group_outputs(seq_group, outputs)
640
641
# Free the finished sequence groups.
642
self.scheduler.free_finished_seq_groups()
vllm/lora/layers.py
+3
-2
@@ -267,12 +267,13 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
267
268
def forward(self, x: torch.Tensor) -> torch.Tensor:
269
added_tokens_mask = x > self.base_layer.org_vocab_size - 1
270
- indices = self.embeddings_indices[1][:self.indices_len[3]].view_as(x)
270
+ embedding_len = self.indices_len[3]
271
+ indices = self.embeddings_indices[1][:embedding_len].view_as(x)
272
full_lora_a_embeddings = F.embedding(
273
x + indices,
274
self.lora_a_stacked_2d,
275
)
275
- indices = self.embeddings_indices[0][:self.indices_len[3]].view_as(x)
276
+ indices = self.embeddings_indices[0][:embedding_len].view_as(x)
277
full_output = self.base_layer.forward(
278
x.add_(indices * added_tokens_mask))
279
vllm/sequence.py
+2
-1
@@ -500,7 +500,8 @@ class SequenceGroup:
500
def get_num_uncomputed_tokens(self) -> int:
501
num_uncomputed_tokens = 0
502
for seq in self.get_seqs():
503
- num_uncomputed_tokens += seq.data.get_num_uncomputed_tokens()
503
+ if not seq.is_finished():
504
+ num_uncomputed_tokens += seq.data.get_num_uncomputed_tokens()
505
return num_uncomputed_tokens
506
507
def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
vllm/worker/model_runner.py
+241
-82
@@ -1,12 +1,14 @@
1
import contextlib
2
import time
3
-from typing import Dict, List, Optional, Set, Tuple
3
+from enum import IntEnum
4
+from typing import Dict, List, NamedTuple, Optional, Set, Tuple
5
6
import numpy as np
7
import torch
8
import torch.nn as nn
9
9
-from vllm.attention import AttentionMetadata, get_attn_backend
10
+from vllm.attention import (AttentionMetadata, AttentionMetadataPerStage,
11
+ get_attn_backend)
12
from vllm.config import (DeviceConfig, LoRAConfig, ModelConfig, ParallelConfig,
13
SchedulerConfig, VisionLanguageConfig)
14
from vllm.distributed import broadcast_tensor_dict, with_pynccl_for_all_reduce
@@ -37,6 +39,66 @@ _BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
39
]
40
41
42
+class PreparePromptMetadata(NamedTuple):
43
+ input_tokens: List[int]
44
+ input_positions: List[int]
45
+ attn_metadata: Optional[AttentionMetadataPerStage]
46
+ prompt_lens: List[int]
47
+ subquery_lens: List[int]
48
+ lora_index_mapping: List[int]
49
+ lora_prompt_mapping: List[int]
50
+ lora_requests: Set[LoRARequest]
51
+ multi_modal_input: Optional[torch.Tensor]
52
+ slot_mapping: List[int]
53
+
54
+ @classmethod
55
+ def empty(cls):
56
+ return PreparePromptMetadata(
57
+ input_tokens=[],
58
+ input_positions=[],
59
+ attn_metadata=None,
60
+ prompt_lens=[],
61
+ subquery_lens=[],
62
+ lora_index_mapping=[],
63
+ lora_prompt_mapping=[],
64
+ lora_requests=set(),
65
+ multi_modal_input=None,
66
+ slot_mapping=[],
67
+ )
68
+
69
+
70
+class PrepareDecodeMetadata(NamedTuple):
71
+ input_tokens: List[int]
72
+ input_positions: List[int]
73
+ attn_metadata: Optional[AttentionMetadata]
74
+ lora_index_mapping: List[int]
75
+ lora_prompt_mapping: List[int]
76
+ lora_requests: Set[LoRARequest]
77
+ slot_mapping: List[int]
78
+
79
+ @classmethod
80
+ def empty(cls):
81
+ return PrepareDecodeMetadata(
82
+ input_tokens=[],
83
+ input_positions=[],
84
+ attn_metadata=None,
85
+ lora_index_mapping=[],
86
+ lora_prompt_mapping=[],
87
+ lora_requests=set(),
88
+ slot_mapping=[],
89
+ )
90
+
91
+
92
+# How batches are constructed.
93
+class BatchType(IntEnum):
94
+ # Every batch is prefill.
95
+ PREFILL = 0
96
+ # Every batch is decode.
97
+ DECODE = 1
98
+ # Batch is a mixture of prefill and decode.
99
+ MIXED = 2
100
+
101
+
102
class ModelRunner:
103
104
def __init__(
@@ -152,10 +214,7 @@ class ModelRunner:
214
def _prepare_prompt(
215
self,
216
seq_group_metadata_list: List[SequenceGroupMetadata],
155
- ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
156
- List[int], List[int], List[int], Set[LoRARequest],
157
- torch.Tensor]:
158
- assert len(seq_group_metadata_list) > 0
217
+ ) -> PreparePromptMetadata:
218
input_tokens: List[int] = []
219
input_positions: List[int] = []
220
slot_mapping: List[int] = []
@@ -169,6 +228,9 @@ class ModelRunner:
228
prefix_block_tables: List[List[int]] = []
229
multi_modal_input_list: List[torch.Tensor] = []
230
231
+ if len(seq_group_metadata_list) == 0:
232
+ return PreparePromptMetadata.empty()
233
+
234
for seq_group_metadata in seq_group_metadata_list:
235
assert seq_group_metadata.is_prompt
236
seq_ids = list(seq_group_metadata.seq_data.keys())
@@ -178,7 +240,8 @@ class ModelRunner:
240
computed_block_nums = seq_group_metadata.computed_block_nums
241
if (self.scheduler_config is not None
242
and self.scheduler_config.chunked_prefill_enabled
181
- and computed_block_nums is not None):
243
+ and not (computed_block_nums is None
244
+ or computed_block_nums == [])):
245
raise RuntimeError(
246
"chunked prefill cannot be used with prefix caching "
247
"now.")
@@ -190,13 +253,8 @@ class ModelRunner:
253
# it contains output tokens.
254
prefill_end = min(seq_data.get_len(),
255
computed_len + token_chunk_size)
193
- # TODO(sang): Rename it after chunked prefill is introduced.
256
prompt_tokens = seq_data.get_token_ids()[computed_len:prefill_end]
195
- prompt_len = len(prompt_tokens)
196
- # Right now, the prefill_end is always same as the length of
197
- # sequence. However, once chunked prefill is introduced, this
198
- # assumption can be changed.
199
- assert prefill_end == seq_data.get_len()
257
+ prompt_len = prefill_end
258
prompt_lens.append(prompt_len)
259
260
# NOTE: This only works for oooooooxxx style attention.
@@ -206,6 +264,14 @@ class ModelRunner:
264
computed_len = len(computed_block_nums) * self.block_size
265
prompt_tokens = prompt_tokens[computed_len:]
266
prefix_block_tables.append(computed_block_nums)
267
+ elif self.scheduler_config.chunked_prefill_enabled:
268
+ if seq_group_metadata.block_tables is not None:
269
+ # Prefill has chunked before.
270
+ block_table = seq_group_metadata.block_tables[seq_id]
271
+ prefix_block_tables.append(block_table)
272
+ else:
273
+ # The first prefill.
274
+ prefix_block_tables.append([])
275
else:
276
prefix_block_tables.append([])
277
# Right now, prefill start is always 0. However, this
@@ -267,20 +333,8 @@ class ModelRunner:
333
334
max_subquery_len = max(subquery_lens)
335
max_prompt_len = max(prompt_lens)
270
- num_prompt_tokens = len(input_tokens)
336
assert max_subquery_len > 0
337
273
- input_tokens = torch.tensor(input_tokens,
274
- dtype=torch.long,
275
- device=self.device)
276
- input_positions = torch.tensor(input_positions,
277
- dtype=torch.long,
278
- device=self.device)
279
- slot_mapping = torch.tensor(slot_mapping,
280
- dtype=torch.long,
281
- device=self.device)
282
- lora_index_mapping = lora_index_mapping
283
-
338
context_lens_tensor = torch.tensor(context_lens,
339
dtype=torch.int,
340
device=self.device)
@@ -332,11 +386,8 @@ class ModelRunner:
386
387
attn_metadata = self.attn_backend.make_metadata(
388
is_prompt=True,
335
- slot_mapping=slot_mapping,
389
prompt_lens=prompt_lens,
390
prompt_lens_tensor=prompt_lens_tensor,
338
- num_prompt_tokens=num_prompt_tokens,
339
- num_generation_tokens=0,
391
max_subquery_len=max_subquery_len,
392
max_context_len=None,
393
max_prompt_len=max_prompt_len,
@@ -345,18 +396,25 @@ class ModelRunner:
396
context_lens=context_lens_tensor,
397
block_tables=block_tables,
398
use_cuda_graph=False,
348
- kv_cache_dtype=self.kv_cache_dtype,
399
)
350
- return (input_tokens, input_positions, attn_metadata, prompt_lens,
351
- subquery_lens, lora_index_mapping, lora_prompt_mapping,
352
- lora_requests, multi_modal_input)
400
+
401
+ return PreparePromptMetadata(
402
+ input_tokens=input_tokens,
403
+ input_positions=input_positions,
404
+ attn_metadata=attn_metadata,
405
+ prompt_lens=prompt_lens,
406
+ subquery_lens=subquery_lens,
407
+ lora_index_mapping=lora_index_mapping,
408
+ lora_prompt_mapping=lora_prompt_mapping,
409
+ lora_requests=lora_requests,
410
+ multi_modal_input=multi_modal_input,
411
+ slot_mapping=slot_mapping,
412
+ )
413
414
def _prepare_decode(
415
self,
416
seq_group_metadata_list: List[SequenceGroupMetadata],
357
- ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
358
- List[int], Set[LoRARequest]]:
359
- assert len(seq_group_metadata_list) > 0
417
+ ) -> PrepareDecodeMetadata:
418
input_tokens: List[int] = []
419
input_positions: List[int] = []
420
slot_mapping: List[int] = []
@@ -366,6 +424,9 @@ class ModelRunner:
424
lora_prompt_mapping: List[int] = []
425
lora_requests: Set[LoRARequest] = set()
426
427
+ if len(seq_group_metadata_list) == 0:
428
+ return PrepareDecodeMetadata.empty()
429
+
430
for seq_group_metadata in seq_group_metadata_list:
431
assert not seq_group_metadata.is_prompt
432
assert seq_group_metadata.token_chunk_size == 1
@@ -424,15 +485,6 @@ class ModelRunner:
485
lora_index_mapping.append(0)
486
batch_size = graph_batch_size
487
427
- input_tokens = torch.tensor(input_tokens,
428
- dtype=torch.long,
429
- device=self.device)
430
- input_positions = torch.tensor(input_positions,
431
- dtype=torch.long,
432
- device=self.device)
433
- slot_mapping = torch.tensor(slot_mapping,
434
- dtype=torch.long,
435
- device=self.device)
488
context_lens = torch.tensor(context_lens,
489
dtype=torch.int,
490
device=self.device)
@@ -440,9 +492,9 @@ class ModelRunner:
492
if use_captured_graph:
493
# When using cuda-graph all these tensors should be
494
# padded.
443
- assert context_lens.shape[0] == input_tokens.shape[0]
444
- assert context_lens.shape[0] == input_positions.shape[0]
445
- assert context_lens.shape[0] == slot_mapping.shape[0]
495
+ assert context_lens.shape[0] == len(input_tokens)
496
+ assert context_lens.shape[0] == len(input_positions)
497
+ assert context_lens.shape[0] == len(slot_mapping)
498
499
# The shape of graph_block_tables is
500
# [max batch size, max context len // block size].
@@ -464,11 +516,8 @@ class ModelRunner:
516
517
attn_metadata = self.attn_backend.make_metadata(
518
is_prompt=False,
467
- slot_mapping=slot_mapping,
519
prompt_lens=None,
520
prompt_lens_tensor=None,
470
- num_prompt_tokens=0,
471
- num_generation_tokens=len(input_tokens),
521
max_subquery_len=None,
522
max_context_len=max_context_len,
523
max_prompt_len=None,
@@ -477,10 +526,16 @@ class ModelRunner:
526
context_lens=context_lens,
527
block_tables=block_tables,
528
use_cuda_graph=use_captured_graph,
480
- kv_cache_dtype=self.kv_cache_dtype,
529
)
482
- return (input_tokens, input_positions, attn_metadata,
483
- lora_index_mapping, lora_prompt_mapping, lora_requests)
530
+ return PrepareDecodeMetadata(
531
+ input_tokens=input_tokens,
532
+ input_positions=input_positions,
533
+ attn_metadata=attn_metadata,
534
+ lora_index_mapping=lora_index_mapping,
535
+ lora_prompt_mapping=lora_prompt_mapping,
536
+ lora_requests=lora_requests,
537
+ slot_mapping=slot_mapping,
538
+ )
539
540
def _prepare_sample(
541
self,
@@ -586,26 +641,66 @@ class ModelRunner:
641
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
642
Set[int], LoRAMapping, torch.Tensor]:
643
if self.is_driver_worker:
589
- # NOTE: We assume that all sequences in the group are all prompts or
590
- # all decodes.
591
- is_prompt = seq_group_metadata_list[0].is_prompt
644
+ prefill_reqs = []
645
+ decode_reqs = []
646
+ for seq_group_meta in seq_group_metadata_list:
647
+ if seq_group_meta.is_prompt:
648
+ prefill_reqs.append(seq_group_meta)
649
+ else:
650
+ decode_reqs.append(seq_group_meta)
651
+
652
# Prepare input tensors.
593
- if is_prompt:
594
- (input_tokens, input_positions, attn_metadata, prompt_lens,
595
- subquery_lens, lora_index_mapping, lora_prompt_mapping,
596
- lora_requests, multi_modal_input
597
- ) = self._prepare_prompt(seq_group_metadata_list)
598
- else:
599
- (input_tokens, input_positions, attn_metadata,
600
- lora_index_mapping, lora_prompt_mapping,
601
- lora_requests) = self._prepare_decode(seq_group_metadata_list)
602
- prompt_lens = []
603
- subquery_lens = None
604
- multi_modal_input = None
653
+ (
654
+ input_tokens,
655
+ input_positions,
656
+ prefill_attn_metadata,
657
+ prompt_lens,
658
+ subquery_lens,
659
+ lora_index_mapping,
660
+ lora_prompt_mapping,
661
+ lora_requests,
662
+ multi_modal_input,
663
+ slot_mapping,
664
+ ) = self._prepare_prompt(prefill_reqs)
665
+ (
666
+ decode_input_tokens,
667
+ decode_input_positions,
668
+ decode_attn_metadata,
669
+ decode_lora_index_mapping,
670
+ decode_lora_prompt_mapping,
671
+ decode_lora_requests,
672
+ decode_slot_mapping,
673
+ ) = self._prepare_decode(decode_reqs)
674
sampling_metadata = self._prepare_sample(seq_group_metadata_list,
675
prompt_lens,
676
subquery_lens)
677
678
+ if not self.scheduler_config.chunked_prefill_enabled:
679
+ assert (len(prefill_reqs) and len(decode_reqs)) == 0
680
+
681
+ num_prefills = len(prompt_lens)
682
+ num_prefill_tokens = len(input_tokens)
683
+ num_decode_tokens = len(decode_input_tokens)
684
+
685
+ # Coalesce tensors. Note that attn_metadata is currently not
686
+ # coalesced for simplicity.
687
+ input_tokens.extend(decode_input_tokens)
688
+ input_positions.extend(decode_input_positions)
689
+ slot_mapping.extend(decode_slot_mapping)
690
+ lora_index_mapping.extend(decode_lora_index_mapping)
691
+ lora_prompt_mapping.extend(decode_lora_prompt_mapping)
692
+ lora_requests.update(decode_lora_requests)
693
+
694
+ input_tokens = torch.tensor(input_tokens,
695
+ dtype=torch.long,
696
+ device=self.device)
697
+ input_positions = torch.tensor(input_positions,
698
+ dtype=torch.long,
699
+ device=self.device)
700
+ slot_mapping = torch.tensor(slot_mapping,
701
+ dtype=torch.long,
702
+ device=self.device)
703
+
704
if self.lora_config:
705
lora_mapping = LoRAMapping(
706
lora_index_mapping,
@@ -615,6 +710,16 @@ class ModelRunner:
710
lora_mapping = None
711
712
# Broadcast the metadata.
713
+ # If batch contains both prefill and decode, it sends 2 broadcasts.
714
+ # If it only contains 1 type, it triggers a single broadcast.
715
+ if (prefill_attn_metadata is not None
716
+ and decode_attn_metadata is not None):
717
+ batch_type = BatchType.MIXED
718
+ elif prefill_attn_metadata is not None:
719
+ batch_type = BatchType.PREFILL
720
+ else:
721
+ batch_type = BatchType.DECODE
722
+
723
metadata_dict = {
724
"input_tokens": input_tokens,
725
"input_positions": input_positions,
@@ -623,19 +728,49 @@ class ModelRunner:
728
"lora_requests": lora_requests,
729
"lora_mapping": lora_mapping,
730
"multi_modal_input": multi_modal_input,
731
+ "num_prefill_tokens": num_prefill_tokens,
732
+ "num_decode_tokens": num_decode_tokens,
733
+ "slot_mapping": slot_mapping,
734
+ "num_prefills": num_prefills,
735
+ "batch_type": batch_type,
736
}
627
- metadata_dict.update(attn_metadata.asdict_zerocopy())
737
+ if prefill_attn_metadata is not None:
738
+ metadata_dict.update(prefill_attn_metadata.asdict_zerocopy())
739
+ else:
740
+ metadata_dict.update(decode_attn_metadata.asdict_zerocopy())
741
broadcast_tensor_dict(metadata_dict, src=0)
742
+
743
+ # Broadcast decode attn metadata for mixed batch type.
744
+ # The additional broadcast costs 300us overhead on 4 A10 GPUs.
745
+ # We can potentially reduce the overhead by coelescing tensors.
746
+ if batch_type == BatchType.MIXED:
747
+ assert decode_attn_metadata is not None
748
+ metadata_dict = decode_attn_metadata.asdict_zerocopy()
749
+ broadcast_tensor_dict(metadata_dict, src=0)
750
else:
751
metadata_dict = broadcast_tensor_dict(src=0)
752
input_tokens = metadata_dict.pop("input_tokens")
753
input_positions = metadata_dict.pop("input_positions")
754
+ slot_mapping = metadata_dict.pop("slot_mapping")
755
+ num_prefills = metadata_dict.pop("num_prefills")
756
selected_token_indices = metadata_dict.pop(
757
"selected_token_indices")
758
lora_mapping = metadata_dict.pop("lora_mapping")
759
lora_requests = metadata_dict.pop("lora_requests")
760
multi_modal_input = metadata_dict.pop("multi_modal_input")
638
- attn_metadata = self.attn_backend.make_metadata(**metadata_dict)
761
+ num_prefill_tokens = metadata_dict.pop("num_prefill_tokens")
762
+ num_decode_tokens = metadata_dict.pop("num_decode_tokens")
763
+ batch_type = metadata_dict.pop("batch_type")
764
+
765
+ # Create an attention metadata.
766
+ prefill_attn_metadata = None
767
+ decode_attn_metadata = None
768
+ if batch_type == BatchType.PREFILL or batch_type == BatchType.MIXED:
769
+ prefill_attn_metadata = self.attn_backend.make_metadata(
770
+ **metadata_dict)
771
+ else:
772
+ decode_attn_metadata = self.attn_backend.make_metadata(
773
+ **metadata_dict)
774
sampling_metadata = SamplingMetadata(
775
seq_groups=None,
776
seq_data=None,
@@ -646,6 +781,23 @@ class ModelRunner:
781
perform_sampling=False,
782
)
783
784
+ # if it is a mixed batch, decode attn_metadata is broadcasted
785
+ # separately.
786
+ if batch_type == BatchType.MIXED:
787
+ metadata_dict = broadcast_tensor_dict(src=0)
788
+ decode_attn_metadata = self.attn_backend.make_metadata(
789
+ **metadata_dict)
790
+
791
+ attn_metadata = AttentionMetadata(
792
+ num_prefills=num_prefills,
793
+ slot_mapping=slot_mapping,
794
+ num_prefill_tokens=num_prefill_tokens,
795
+ num_decode_tokens=num_decode_tokens,
796
+ prefill_metadata=prefill_attn_metadata,
797
+ decode_metadata=decode_attn_metadata,
798
+ kv_cache_dtype=self.kv_cache_dtype,
799
+ )
800
+
801
return (input_tokens, input_positions, attn_metadata,
802
sampling_metadata, lora_requests, lora_mapping,
803
multi_modal_input)
@@ -663,8 +815,10 @@ class ModelRunner:
815
if self.lora_config:
816
self.set_active_loras(lora_requests, lora_mapping)
817
666
- # Execute the model.
667
- if attn_metadata.use_cuda_graph:
818
+ # Currently cuda graph is only supported by the decode phase.
819
+ prefill_meta = attn_metadata.prefill_metadata
820
+ decode_meta = attn_metadata.decode_metadata
821
+ if prefill_meta is None and decode_meta.use_cuda_graph:
822
graph_batch_size = input_tokens.shape[0]
823
model_executable = self.graph_runners[graph_batch_size]
824
else:
@@ -842,13 +996,10 @@ class ModelRunner:
996
# memory usage of CUDA graph.
997
for batch_size in reversed(batch_size_capture_list):
998
# Create dummy attn_metadata.
845
- attn_metadata = self.attn_backend.make_metadata(
999
+ decode_metadata = self.attn_backend.make_metadata(
1000
is_prompt=False,
847
- slot_mapping=slot_mapping[:batch_size],
1001
prompt_lens=None,
1002
prompt_lens_tensor=None,
850
- num_prompt_tokens=0,
851
- num_generation_tokens=batch_size,
1003
max_subquery_len=None,
1004
max_context_len=self.max_context_len_to_capture,
1005
max_prompt_len=None,
@@ -857,6 +1008,14 @@ class ModelRunner:
1008
context_lens=context_lens[:batch_size],
1009
block_tables=block_tables[:batch_size],
1010
use_cuda_graph=True,
1011
+ )
1012
+ attn_metadata = AttentionMetadata(
1013
+ num_prefills=0,
1014
+ num_prefill_tokens=0,
1015
+ num_decode_tokens=batch_size,
1016
+ slot_mapping=slot_mapping[:batch_size],
1017
+ prefill_metadata=None,
1018
+ decode_metadata=decode_metadata,
1019
kv_cache_dtype=self.kv_cache_dtype,
1020
)
1021
@@ -950,8 +1109,8 @@ class CUDAGraphRunner:
1109
"positions": positions,
1110
"kv_caches": kv_caches,
1111
"slot_mapping": attn_metadata.slot_mapping,
953
- "context_lens": attn_metadata.context_lens,
954
- "block_tables": attn_metadata.block_tables,
1112
+ "context_lens": attn_metadata.decode_metadata.context_lens,
1113
+ "block_tables": attn_metadata.decode_metadata.block_tables,
1114
}
1115
self.output_buffers = {"hidden_states": hidden_states}
1116
return
@@ -972,10 +1131,10 @@ class CUDAGraphRunner:
1131
self.input_buffers["positions"].copy_(positions, non_blocking=True)
1132
self.input_buffers["slot_mapping"].copy_(attn_metadata.slot_mapping,
1133
non_blocking=True)
975
- self.input_buffers["context_lens"].copy_(attn_metadata.context_lens,
976
- non_blocking=True)
977
- self.input_buffers["block_tables"].copy_(attn_metadata.block_tables,
978
- non_blocking=True)
1134
+ self.input_buffers["context_lens"].copy_(
1135
+ attn_metadata.decode_metadata.context_lens, non_blocking=True)
1136
+ self.input_buffers["block_tables"].copy_(
1137
+ attn_metadata.decode_metadata.block_tables, non_blocking=True)
1138
# Run the graph.
1139
self.graph.replay()
1140