A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[1/n][Chunked Prefill] Refactor input query shapes (#3236)
SangBin Cho committed
Mar 21, 2024 at 06:46 UTC
6e435de766c7749b214b637ac58570a221006c95
18 files changed
+575
-259
.buildkite/test-pipeline.yaml
+2
-2
@@ -47,7 +47,7 @@ steps:
47
- pytest -v -s prefix_caching
48
49
- label: Samplers Test
50
- command: pytest -v -s samplers --forked
50
+ command: pytest -v -s samplers
51
52
- label: Worker Test
53
command: pytest -v -s worker
@@ -56,7 +56,7 @@ steps:
56
command: pytest -v -s spec_decode
57
58
- label: LoRA Test %N
59
- command: pytest -v -s lora --forked --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
59
+ command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
60
parallelism: 4
61
62
- label: Metrics Test
tests/basic_correctness/test_basic_correctness.py
+3
-1
@@ -13,6 +13,7 @@ MODELS = [
13
@pytest.mark.parametrize("model", MODELS)
14
@pytest.mark.parametrize("dtype", ["half"])
15
@pytest.mark.parametrize("max_tokens", [5])
16
+@pytest.mark.parametrize("enforce_eager", [False, True])
17
def test_models(
18
hf_runner,
19
vllm_runner,
@@ -20,12 +21,13 @@ def test_models(
21
model: str,
22
dtype: str,
23
max_tokens: int,
24
+ enforce_eager: bool,
25
) -> None:
26
hf_model = hf_runner(model, dtype=dtype)
27
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
28
del hf_model
29
28
- vllm_model = vllm_runner(model, dtype=dtype)
30
+ vllm_model = vllm_runner(model, dtype=dtype, enforce_eager=enforce_eager)
31
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
32
del vllm_model
33
tests/core/test_scheduler.py
+9
-9
@@ -10,7 +10,7 @@ from .utils import create_dummy_prompt
10
11
def test_scheduler_add_seq_group():
12
block_size = 4
13
- scheduler_config = SchedulerConfig(100, 64, 1, 256)
13
+ scheduler_config = SchedulerConfig(100, 64, 1)
14
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
15
cache_config.num_cpu_blocks = 4
16
cache_config.num_gpu_blocks = 4
@@ -26,7 +26,7 @@ def test_scheduler_add_seq_group():
26
27
def test_scheduler_abort_seq_group():
28
block_size = 4
29
- scheduler_config = SchedulerConfig(100, 64, 1, 256)
29
+ scheduler_config = SchedulerConfig(100, 64, 1)
30
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
31
cache_config.num_cpu_blocks = 4
32
cache_config.num_gpu_blocks = 4
@@ -50,7 +50,7 @@ def test_scheduler_schedule_simple():
50
block_size = 4
51
num_seq_group = 4
52
max_model_len = 16
53
- scheduler_config = SchedulerConfig(64, num_seq_group, max_model_len, 256)
53
+ scheduler_config = SchedulerConfig(64, num_seq_group, max_model_len)
54
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
55
cache_config.num_cpu_blocks = 8
56
cache_config.num_gpu_blocks = 8
@@ -64,10 +64,10 @@ def test_scheduler_schedule_simple():
64
running.append(seq_group)
65
66
# Schedule seq groups prompts.
67
+ num_tokens = block_size * num_seq_group
68
seq_group_meta, out = scheduler.schedule()
69
assert set(out.scheduled_seq_groups) == set(running)
69
- assert out.num_batched_tokens == num_seq_group * seq_group.get_seqs(
70
- )[0].get_len()
70
+ assert out.num_batched_tokens == num_tokens
71
assert (not out.blocks_to_copy and not out.blocks_to_swap_in
72
and not out.blocks_to_swap_out)
73
assert len(seq_group_meta) == num_seq_group
@@ -84,7 +84,7 @@ def test_scheduler_schedule_simple():
84
def test_scheduler_schedule_preempt_abort():
85
block_size = 4
86
max_model_len = 16
87
- scheduler_config = SchedulerConfig(64, 2, max_model_len, 256)
87
+ scheduler_config = SchedulerConfig(64, 2, max_model_len)
88
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
89
cache_config.num_cpu_blocks = 2
90
cache_config.num_gpu_blocks = 2
@@ -99,7 +99,7 @@ def test_scheduler_schedule_preempt_abort():
99
# Schedule seq groups prompts.
100
seq_group_meta, out = scheduler.schedule()
101
assert out.scheduled_seq_groups == [seq_group_a, seq_group_b]
102
- assert out.num_batched_tokens == seq_group_a.get_seqs()[0].get_len() * 2
102
+ assert out.num_batched_tokens == block_size * 2 # seq_a and seq_b
103
assert (not out.blocks_to_copy and not out.blocks_to_swap_in
104
and not out.blocks_to_swap_out)
105
assert len(seq_group_meta) == 2
@@ -124,7 +124,7 @@ def test_scheduler_schedule_preempt_abort():
124
scheduler.abort_seq_group("1")
125
seq_group_meta, out = scheduler.schedule()
126
assert out.scheduled_seq_groups == [seq_group_b]
127
- assert out.num_batched_tokens == seq_group_b.get_seqs()[0].get_len()
127
+ assert out.num_batched_tokens == 5 # 4 prompt + 1 generation.
128
assert (not out.blocks_to_copy and not out.blocks_to_swap_in
129
and not out.blocks_to_swap_out)
130
assert len(seq_group_meta) == 1
@@ -136,7 +136,7 @@ def test_scheduler_max_seqs():
136
num_seq_group = 4
137
max_seq_group = 2
138
max_model_len = 16
139
- scheduler_config = SchedulerConfig(64, max_seq_group, max_model_len, 256)
139
+ scheduler_config = SchedulerConfig(64, max_seq_group, max_model_len)
140
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
141
cache_config.num_cpu_blocks = 8
142
cache_config.num_gpu_blocks = 8
tests/lora/test_worker.py
+1
-1
@@ -25,7 +25,7 @@ def test_worker_apply_lora(sql_lora_files):
25
revision=None,
26
),
27
parallel_config=ParallelConfig(1, 1, False),
28
- scheduler_config=SchedulerConfig(32, 32, 32, 256),
28
+ scheduler_config=SchedulerConfig(32, 32, 32),
29
device_config=DeviceConfig("cuda"),
30
local_rank=0,
31
rank=0,
tests/spec_decode/test_multi_step_worker.py
+2
-2
@@ -92,8 +92,8 @@ def test_same_output_for_single_step():
92
num_gpu_blocks,
93
seed,
94
)
95
- multi_step_worker.model_runner = worker.model_runner
96
- multi_step_worker.cache_engine = worker.cache_engine
95
+ # multi_step_worker.model_runner = worker.model_runner
96
+ # multi_step_worker.cache_engine = worker.cache_engine
97
98
num_steps = 1
99
tests/worker/test_model_runner.py
+153
-8
@@ -1,8 +1,13 @@
1
import random
2
import torch
3
4
+from vllm.config import ModelConfig
5
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
5
-from vllm.worker.model_runner import ModelRunner
6
+from vllm.worker.model_runner import ModelRunner, _BATCH_SIZE_ALIGNMENT
7
+
8
+
9
+def get_aligned_size(batch_size: int, alignment: int):
10
+ return ((batch_size + alignment - 1) // alignment * alignment)
11
12
13
def test_prepare_prompt():
@@ -12,6 +17,7 @@ def test_prepare_prompt():
17
batch_size = random.randint(1, 256)
18
prompt_lens = []
19
seq_group_metadata_list = []
20
+ block_tables = {0: [1]}
21
for i in range(batch_size):
22
# make sure all tokens fit into one block
23
prompt_len = i % (model_runner.block_size - 1) + 1
@@ -23,26 +29,165 @@ def test_prepare_prompt():
29
is_prompt=True,
30
seq_data={0: SequenceData(seq_data)},
31
sampling_params=SamplingParams(temperature=0),
26
- block_tables={0: [1]},
32
+ block_tables=block_tables,
33
))
34
35
expected_selected_token_indices = []
36
selected_token_start_idx = 0
31
- max_seq_len = max(prompt_lens)
37
for prompt_len in prompt_lens:
38
expected_selected_token_indices.append(selected_token_start_idx +
39
prompt_len - 1)
35
- selected_token_start_idx += max_seq_len
36
- input_tokens, input_positions, _, return_prompt_lens, _, _, _, _ = (
37
- model_runner._prepare_prompt(seq_group_metadata_list))
40
+ selected_token_start_idx += prompt_len
41
+ (input_tokens, input_positions, input_metadata, return_prompt_lens, _, _,
42
+ _, _) = (model_runner._prepare_prompt(seq_group_metadata_list))
43
assert return_prompt_lens == prompt_lens
44
+
45
+ # Verify input metadata is correct for prompts.
46
+ device = model_runner.device
47
+ assert input_metadata.is_prompt is True
48
+ assert torch.allclose(input_metadata.prompt_lens_tensor,
49
+ torch.tensor(prompt_lens, device=device))
50
+ assert input_metadata.prompt_lens == prompt_lens
51
+ assert input_metadata.num_prompt_tokens == sum(prompt_lens)
52
+ assert input_metadata.num_generation_tokens == 0
53
+ assert input_metadata.max_seq_len == max(prompt_lens)
54
+
55
+ # Test subquery start locs.
56
+ start_idx = 0
57
+ start_loc = [start_idx]
58
+ for prompt_len in prompt_lens:
59
+ start_idx += prompt_len
60
+ start_loc.append(start_idx)
61
+ assert torch.allclose(
62
+ input_metadata.subquery_start_loc,
63
+ torch.tensor(start_loc, dtype=torch.int32, device=device))
64
+
65
+ # Test seq start locs. Note that for normal prefill it is
66
+ # equivalent to subquery_start_loc.
67
+ start_idx = 0
68
+ seq_start_loc = [start_idx]
69
+ for prompt_len in prompt_lens:
70
+ start_idx += prompt_len
71
+ seq_start_loc.append(start_idx)
72
+
73
+ assert torch.allclose(
74
+ input_metadata.seq_start_loc,
75
+ torch.tensor(start_loc, dtype=torch.int32, device=device))
76
+ assert input_metadata.max_context_len is None
77
+ assert torch.allclose(
78
+ input_metadata.context_lens,
79
+ torch.zeros(input_metadata.context_lens.shape[0],
80
+ dtype=torch.int,
81
+ device=device))
82
+
83
+ expected = torch.tensor([[] for _ in range(len(seq_group_metadata_list))],
84
+ dtype=torch.int32,
85
+ device=model_runner.device)
86
+ assert torch.allclose(input_metadata.block_tables, expected)
87
+ # Cuda graph should not be used for prerill.
88
+ assert input_metadata.use_cuda_graph is False
89
+ assert input_metadata.kv_cache_dtype == "auto"
90
+
91
+ assert input_tokens.shape == (sum(prompt_lens), )
92
+ assert input_positions.shape == (sum(prompt_lens), )
93
+ torch.testing.assert_close(input_tokens, input_positions)
94
+
95
sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
96
prompt_lens,
97
subquery_lens=prompt_lens)
42
- assert input_tokens.shape == (batch_size, max_seq_len)
43
- assert input_positions.shape == (batch_size, max_seq_len)
98
+ assert input_tokens.shape == (sum(prompt_lens), )
99
+ assert input_positions.shape == (sum(prompt_lens), )
100
+ actual = sampling_metadata.selected_token_indices
101
+ expected = torch.tensor(expected_selected_token_indices,
102
+ device=actual.device,
103
+ dtype=actual.dtype)
104
+ torch.testing.assert_close(actual, expected)
105
+ torch.testing.assert_close(input_tokens, input_positions)
106
+
107
+ actual = sampling_metadata.selected_token_indices
108
+ expected = torch.tensor(expected_selected_token_indices,
109
+ device=actual.device,
110
+ dtype=actual.dtype)
111
+ torch.testing.assert_close(actual, expected)
112
+
113
+
114
+def test_prepare_decode_cuda_graph():
115
+ model_config = ModelConfig(
116
+ "facebook/opt-125m",
117
+ "facebook/opt-125m",
118
+ tokenizer_mode="auto",
119
+ trust_remote_code=False,
120
+ download_dir=None,
121
+ load_format="dummy",
122
+ seed=0,
123
+ dtype="float16",
124
+ revision=None,
125
+ enforce_eager=False,
126
+ )
127
+ model_runner = ModelRunner(model_config, None, None, None, None)
128
+ model_runner.set_block_size(16)
129
+
130
+ batch_size = random.randint(1, 256)
131
+ prompt_lens = []
132
+ seq_group_metadata_list = []
133
+ for i in range(batch_size):
134
+ # make sure all tokens fit into one block
135
+ prompt_len = i % (model_runner.block_size - 1) + 1
136
+ prompt_lens.append(prompt_len)
137
+ seq_data = list(range(prompt_len))
138
+ seq_group_metadata_list.append(
139
+ SequenceGroupMetadata(
140
+ request_id=f"test_{i}",
141
+ is_prompt=False,
142
+ seq_data={0: SequenceData(seq_data)},
143
+ sampling_params=SamplingParams(temperature=0),
144
+ block_tables={0: [1]},
145
+ ))
146
+
147
+ input_tokens, input_positions, input_metadata, _, _, _ = (
148
+ model_runner._prepare_decode(seq_group_metadata_list))
149
+
150
+ # Verify input metadata is correct for prompts.
151
+ device = model_runner.device
152
+ assert input_metadata.is_prompt is False
153
+ assert input_metadata.prompt_lens is None
154
+ assert input_metadata.num_prompt_tokens == 0
155
+ assert input_metadata.num_generation_tokens == (get_aligned_size(
156
+ len(seq_group_metadata_list), _BATCH_SIZE_ALIGNMENT))
157
+ assert input_metadata.max_seq_len is None
158
+ assert input_metadata.subquery_start_loc is None
159
+ assert input_metadata.seq_start_loc is None
160
+ assert input_metadata.max_context_len == max(prompt_lens)
161
+ assert torch.allclose(
162
+ input_metadata.context_lens[:len(prompt_lens)],
163
+ torch.tensor(prompt_lens, dtype=torch.int, device=device))
164
+
165
+ # block table's first index corresponds to each batch, meaning in
166
+ # decoding it is each token.
167
+ assert input_metadata.block_tables.shape[0] == len(input_tokens)
168
+ # Block table's second dim correspondsd to each token's block number.
169
+ # It is padded up to
170
+ assert input_metadata.block_tables.shape[1] == (
171
+ model_runner.get_max_block_per_batch())
172
+ # Cuda graph should not be used for prerill.
173
+ assert input_metadata.use_cuda_graph is True
174
+ assert input_metadata.kv_cache_dtype == "auto"
175
+
176
+ assert input_tokens.shape == (get_aligned_size(
177
+ len(seq_group_metadata_list), _BATCH_SIZE_ALIGNMENT), )
178
+ assert input_positions.shape == (get_aligned_size(
179
+ len(seq_group_metadata_list), _BATCH_SIZE_ALIGNMENT), )
180
torch.testing.assert_close(input_tokens, input_positions)
181
182
+ # Verify Sampling
183
+ expected_selected_token_indices = []
184
+ selected_token_start_idx = 0
185
+ for prompt_len in prompt_lens:
186
+ expected_selected_token_indices.append(selected_token_start_idx)
187
+ selected_token_start_idx += 1
188
+ sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
189
+ prompt_lens,
190
+ subquery_lens=prompt_lens)
191
actual = sampling_metadata.selected_token_indices
192
expected = torch.tensor(expected_selected_token_indices,
193
device=actual.device,
vllm/config.py
-3
@@ -535,7 +535,6 @@ class SchedulerConfig:
535
iteration.
536
max_model_len: Maximum length of a sequence (including prompt
537
and generated text).
538
- max_paddings: Maximum number of paddings to be added to a batch.
538
"""
539
540
def __init__(
@@ -543,7 +542,6 @@ class SchedulerConfig:
542
max_num_batched_tokens: Optional[int],
543
max_num_seqs: int,
544
max_model_len: int,
546
- max_paddings: int,
545
) -> None:
546
if max_num_batched_tokens is not None:
547
self.max_num_batched_tokens = max_num_batched_tokens
@@ -553,7 +551,6 @@ class SchedulerConfig:
551
self.max_num_batched_tokens = max(max_model_len, 2048)
552
self.max_num_seqs = max_num_seqs
553
self.max_model_len = max_model_len
556
- self.max_paddings = max_paddings
554
self._verify_args()
555
556
def _verify_args(self) -> None:
vllm/core/scheduler.py
+3
-10
@@ -173,12 +173,12 @@ class Scheduler:
173
curr_loras = set(
174
seq_group.lora_int_id
175
for seq_group in self.running) if self.lora_enabled else None
176
- seq_lens: List[int] = []
176
177
# Optimization: We do not sort the waiting queue since the preempted
178
# sequence groups are added to the front and the new sequence groups
179
# are added to the back.
180
leftover_waiting_sequences = deque()
181
+ num_batched_tokens = 0
182
while self.waiting:
183
seq_group = self.waiting[0]
184
waiting_seqs = seq_group.get_seqs(
@@ -223,8 +223,7 @@ class Scheduler:
223
continue
224
225
# If the number of batched tokens exceeds the limit, stop.
226
- new_seq_lens = seq_lens + [num_prompt_tokens]
227
- num_batched_tokens = len(new_seq_lens) * max(new_seq_lens)
226
+ num_batched_tokens += num_prompt_tokens
227
if (num_batched_tokens >
228
self.scheduler_config.max_num_batched_tokens):
229
break
@@ -236,11 +235,6 @@ class Scheduler:
235
self.scheduler_config.max_num_seqs):
236
break
237
239
- num_paddings = num_batched_tokens - sum(new_seq_lens)
240
- if num_paddings > self.scheduler_config.max_paddings:
241
- break
242
- seq_lens = new_seq_lens
243
-
238
if lora_int_id > 0:
239
curr_loras.add(lora_int_id)
240
self.waiting.popleft()
@@ -255,8 +249,7 @@ class Scheduler:
249
scheduler_outputs = SchedulerOutputs(
250
scheduled_seq_groups=scheduled,
251
prompt_run=True,
258
- num_batched_tokens=len(seq_lens) *
259
- max(seq_lens) if seq_lens else 0,
252
+ num_batched_tokens=num_batched_tokens,
253
blocks_to_swap_in=blocks_to_swap_in,
254
blocks_to_swap_out=blocks_to_swap_out,
255
blocks_to_copy=blocks_to_copy,
vllm/engine/arg_utils.py
+1
-7
@@ -31,7 +31,6 @@ class EngineArgs:
31
gpu_memory_utilization: float = 0.90
32
max_num_batched_tokens: Optional[int] = None
33
max_num_seqs: int = 256
34
- max_paddings: int = 256
34
max_logprobs: int = 5 # OpenAI default value
35
disable_log_stats: bool = False
36
revision: Optional[str] = None
@@ -213,10 +212,6 @@ class EngineArgs:
212
type=int,
213
default=EngineArgs.max_num_seqs,
214
help='maximum number of sequences per iteration')
216
- parser.add_argument('--max-paddings',
217
- type=int,
218
- default=EngineArgs.max_paddings,
219
- help='maximum number of paddings in a batch')
215
parser.add_argument(
216
'--max-logprobs',
217
type=int,
@@ -347,8 +342,7 @@ class EngineArgs:
342
), self.ray_workers_use_nsight)
343
scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
344
self.max_num_seqs,
350
- model_config.max_model_len,
351
- self.max_paddings)
345
+ model_config.max_model_len)
346
lora_config = LoRAConfig(
347
max_lora_rank=self.max_lora_rank,
348
max_loras=self.max_loras,
vllm/engine/llm_engine.py
-1
@@ -561,7 +561,6 @@ class LLMEngine:
561
# Log stats.
562
if self.log_stats:
563
self.stat_logger.log(self._get_stats(scheduler_outputs))
564
-
564
return request_outputs
565
566
def step(self) -> List[RequestOutput]:
vllm/model_executor/input_metadata.py
+69
-13
@@ -1,36 +1,92 @@
1
from dataclasses import dataclass, fields
2
-from typing import Optional, Any, Dict
2
+from typing import Optional, List, Any, Dict
3
4
import torch
5
+from xformers.ops.fmha.attn_bias import AttentionBias
6
7
8
@dataclass
9
class InputMetadata:
10
"""Metadata for input sequences. Used in PagedAttention.
11
11
- Args:
12
- prompt_lens: Lengths of prompts.
13
- slot_mapping: The address to write the new KV to of each token.
14
- max_context_len: The maximum context length.
15
- context_lens: the length of attention context for each sequence.
16
- block_tables: The block tables. (Seq id -> list of physical block)
17
- kv_cache_dtype: Data type to store kv cache.
12
+ NOTE: Any python object stored here is not updated when it is
13
+ cuda-graph replayed. If you have values that need to be changed
14
+ dynamically, it should be stored in tensor. The tensor has to be
15
+ updated from `CUDAGraphRunner.forward` API.
16
"""
19
-
17
+ # Currently, input sequences can only contain all prompts
18
+ # or all decoding. True if all sequences are prompts.
19
is_prompt: bool
20
+ # (num_tokens,). The indices of the token slots that input tokens will be
21
+ # stored into. E.g., if `slot_mapping` is [35, 2, 17] and the block size
22
+ # is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot
23
+ # in block 0, and 1st slot in block 1, respectively.
24
slot_mapping: torch.Tensor
22
- prompt_lens: Optional[torch.Tensor]
23
- max_seq_len: Optional[int]
24
- start_loc: Optional[torch.Tensor]
25
+ # (batch_size,). The prompt length per sequence. None if it is a decoding.
26
+ prompt_lens: Optional[List[int]]
27
+ # prompt_lens stored as a tensor.
28
+ prompt_lens_tensor: Optional[torch.Tensor]
29
+ # The number of prompt tokens. Doesn't include padding.
30
+ num_prompt_tokens: int
31
+ # The number of generation tokens. Doesn't include padding.
32
+ num_generation_tokens: int
33
+ """
34
+ Definition of context_len, subquery_len, and seqlen.
35
+ |---------- N-1 iteration --------|
36
+ |---------------- N iteration ---------------------|
37
+ |- tokenA -|......................|-- newTokens ---|
38
+ |---------- context_len ----------|
39
+ |-------------------- seqlen ----------------------|
40
+ |- subquery_len -|
41
+
42
+ WARNING: context_len has different definition depending on if it is
43
+ prefill vs decoding. When it is prefill, it doesn't include new
44
+ tokens. When it is for decoding, it includes a new token.
45
+ """
46
+
47
+ # Maximum subquery length in the batch.
48
+ max_subquery_len: Optional[int]
49
+ # Maximum context length in the batch.
50
max_context_len: Optional[int]
51
+ # FIXME: It is for flash attn.
52
+ # Maximum sequence length in the batch.
53
+ max_seq_len: Optional[int]
54
+ # (batch_size + 1,). The cumulative subquery lengths of the sequences in
55
+ # the batch, used to index into subquery. E.g., if the subquery length
56
+ # is [4, 6], it is [0, 4, 10].
57
+ subquery_start_loc: Optional[torch.Tensor]
58
+ # FIXME: It is for flash attn.
59
+ # (batch_size + 1,). The cumulative sequence lengths of the sequences in
60
+ # the batch, used to index into sequence. E.g., if the sequence length is
61
+ # [4, 6], it is [0, 4, 10].
62
+ seq_start_loc: Optional[torch.Tensor]
63
+ # (batch_size,). The length of context (tokens stored in KV cache) per
64
+ # sequence. WARNING: When it is a prefill request, it doesn't include new
65
+ # tokens. When it is for decoding, it includes a new token.
66
context_lens: Optional[torch.Tensor]
67
+ # (batch_size, max_blocks_per_seq).
68
+ # Block addresses per sequence. (Seq id -> list of physical block)
69
+ # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks
70
+ # in the kv cache. Each block can contain up to block_size tokens.
71
+ # 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph
72
+ # captured.
73
block_tables: Optional[torch.Tensor]
74
+ # Whether or not if cuda graph is enabled.
75
+ # Cuda-graph is currently enabled for decoding only.
76
use_cuda_graph: bool
77
kv_cache_dtype: str
78
79
def __post_init__(self):
80
+ # Set during the execution of the first attention op.
81
+ # It is a list because it is needed to set per prompt
82
+ # when alibi slopes is used. It is because of the limitation
83
+ # from xformer API.
84
# will not appear in the __repr__ and __init__
33
- self.attn_bias = None
85
+ self.attn_bias: Optional[List[AttentionBias]] = None
86
+
87
+ # Cuda graph is only used for decoding now.
88
+ if self.use_cuda_graph:
89
+ assert self.num_prompt_tokens == 0
90
91
def asdict_zerocopy(self) -> Dict[str, Any]:
92
"""Similar to dataclasses.asdict, but avoids deepcopying."""
vllm/model_executor/layers/activation.py
+2
-2
@@ -20,8 +20,8 @@ class SiluAndMul(nn.Module):
20
The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[-1] // 2.
21
22
Shapes:
23
- x: (batch_size, seq_len, 2 * d) or (num_tokens, 2 * d)
24
- return: (batch_size, seq_len, d) or (num_tokens, d)
23
+ x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d)
24
+ return: (num_tokens, d) or (batch_size, seq_len, d)
25
"""
26
27
def _forward(self, x: torch.Tensor) -> torch.Tensor:
vllm/model_executor/layers/attention/attention.py
+2
-1
@@ -17,11 +17,12 @@ class Attention(nn.Module):
17
18
This class takes query, key, and value tensors as input. The input tensors
19
can either contain prompt tokens or generation tokens.
20
+
21
The class does the following:
22
23
1. Store the input key and value tensors in the KV cache.
24
2. Perform (multi-head/multi-query/grouped-query) attention.
24
- 3. Return the output tensor.
25
+ 3. Output the output tensor.
26
"""
27
28
def __init__(
vllm/model_executor/layers/attention/backends/flash_attn.py
+32
-14
@@ -1,7 +1,7 @@
1
"""Attention layer with Flash and PagedAttention."""
2
from typing import List, Optional
3
4
-from flash_attn import flash_attn_func
4
+from flash_attn import flash_attn_varlen_func
5
import torch
6
7
from vllm.model_executor.input_metadata import InputMetadata
@@ -10,6 +10,21 @@ from vllm.model_executor.layers.attention.ops.paged_attn import (
10
11
12
class FlashAttentionBackend:
13
+ """
14
+ If the input tensors contain prompt tokens, the layout is as follows:
15
+ |<--------------- num_prompt_tokens -------------->|
16
+ |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->|
17
+
18
+ Otherwise, the layout is as follows:
19
+ |<------------------ num_generation_tokens (M) ----------------->|
20
+ |<--generation_0-->|..........|<--generation_M-1-->|<--padding-->|
21
+
22
+ Generation tokens can contain padding when cuda-graph is used.
23
+ Currently, prompt tokens don't contain any padding.
24
+
25
+ The prompts might have different lengths, while the generation tokens
26
+ always have length 1.
27
+ """
28
29
def __init__(
30
self,
@@ -52,18 +67,18 @@ class FlashAttentionBackend:
67
"""Forward pass with FlashAttention and PagedAttention.
68
69
Args:
55
- query: shape = [batch_size, seq_len, num_heads * head_size]
56
- key: shape = [batch_size, seq_len, num_kv_heads * head_size]
57
- value: shape = [batch_size, seq_len, num_kv_heads * head_size]
70
+ query: shape = [num_tokens, num_heads * head_size]
71
+ key: shape = [num_tokens, num_kv_heads * head_size]
72
+ value: shape = [num_tokens, num_kv_heads * head_size]
73
key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
74
block_size, x]
75
value_cache: shape = [num_blocks, num_kv_heads, head_size,
76
block_size]
77
input_metadata: metadata for the inputs.
78
Returns:
64
- shape = [batch_size, seq_len, num_heads * head_size]
79
+ shape = [num_tokens, num_heads * head_size]
80
"""
66
- batch_size, seq_len, hidden_size = query.shape
81
+ num_tokens, hidden_size = query.shape
82
# Reshape the query, key, and value tensors.
83
query = query.view(-1, self.num_heads, self.head_size)
84
key = key.view(-1, self.num_kv_heads, self.head_size)
@@ -82,13 +97,16 @@ class FlashAttentionBackend:
97
if (key_cache is None or value_cache is None
98
or input_metadata.block_tables.numel() == 0):
99
# normal attention
85
- query = query.unflatten(0, (batch_size, seq_len))
86
- key = key.unflatten(0, (batch_size, seq_len))
87
- value = value.unflatten(0, (batch_size, seq_len))
88
- output = flash_attn_func(
89
- query,
90
- key,
91
- value,
100
+ # When block_tables are not filled, it means q and k are the
101
+ # prompt, and they have the same length.
102
+ output = flash_attn_varlen_func(
103
+ q=query,
104
+ k=key,
105
+ v=value,
106
+ cu_seqlens_q=input_metadata.seq_start_loc,
107
+ cu_seqlens_k=input_metadata.seq_start_loc,
108
+ max_seqlen_q=input_metadata.max_seq_len,
109
+ max_seqlen_k=input_metadata.max_seq_len,
110
softmax_scale=self.scale,
111
causal=True,
112
window_size=self.sliding_window,
@@ -118,4 +136,4 @@ class FlashAttentionBackend:
136
)
137
138
# Reshape the output tensor.
121
- return output.view(batch_size, seq_len, hidden_size)
139
+ return output.view(num_tokens, hidden_size)
vllm/model_executor/layers/attention/backends/xformers.py
+147
-85
@@ -14,6 +14,21 @@ from vllm.utils import is_hip
14
15
16
class XFormersBackend:
17
+ """
18
+ If the input tensors contain prompt tokens, the layout is as follows:
19
+ |<--------------- num_prompt_tokens --------------->|
20
+ |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1--->|
21
+
22
+ Otherwise, the layout is as follows:
23
+ |<------------------ num_generation_tokens (M) ----------------->|
24
+ |<--generation_0-->|..........|<--generation_M-1-->|<--padding-->|
25
+
26
+ Generation tokens can contain padding when cuda-graph is used.
27
+ Currently, prompt tokens don't contain any padding.
28
+
29
+ The prompts might have different lengths, while the generation tokens
30
+ always have length 1.
31
+ """
32
33
def __init__(
34
self,
@@ -55,19 +70,18 @@ class XFormersBackend:
70
"""Forward pass with xFormers and PagedAttention.
71
72
Args:
58
- query: shape = [batch_size, seq_len, num_heads * head_size]
59
- key: shape = [batch_size, seq_len, num_kv_heads * head_size]
60
- value: shape = [batch_size, seq_len, num_kv_heads * head_size]
73
+ query: shape = [num_tokens, num_heads * head_size]
74
+ key: shape = [num_tokens, num_kv_heads * head_size]
75
+ value: shape = [num_tokens, num_kv_heads * head_size]
76
key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
77
block_size, x]
78
value_cache: shape = [num_blocks, num_kv_heads, head_size,
79
block_size]
80
input_metadata: metadata for the inputs.
81
Returns:
67
- shape = [batch_size, seq_len, num_heads * head_size]
82
+ shape = [num_tokens, num_heads * head_size]
83
"""
69
- batch_size, seq_len, hidden_size = query.shape
70
- # Reshape the query, key, and value tensors.
84
+ num_tokens, hidden_size = query.shape
85
query = query.view(-1, self.num_heads, self.head_size)
86
key = key.view(-1, self.num_kv_heads, self.head_size)
87
value = value.view(-1, self.num_kv_heads, self.head_size)
@@ -82,9 +96,10 @@ class XFormersBackend:
96
97
if input_metadata.is_prompt:
98
# Prompt run.
99
+ # key_cache and value_cache are None when it is a profiling run.
100
+ # block tables are empty if the prompt has never been computed.
101
if (key_cache is None or value_cache is None
102
or input_metadata.block_tables.numel() == 0):
87
- # normal attention
103
if self.num_kv_heads != self.num_heads:
104
# As of Nov 2023, xformers only supports MHA. For MQA/GQA,
105
# project the key and value tensors to the desired number of
@@ -103,61 +118,33 @@ class XFormersBackend:
118
self.num_queries_per_kv,
119
value.shape[-1])
120
106
- # Set attention bias if not provided. This typically happens at
107
- # the very attention layer of every iteration.
108
- # FIXME(woosuk): This is a hack.
109
- if input_metadata.attn_bias is None:
110
- if self.alibi_slopes is None:
111
- attn_bias = BlockDiagonalCausalMask.from_seqlens(
112
- [seq_len] * batch_size)
113
- if self.sliding_window is not None:
114
- attn_bias = attn_bias.make_local_attention(
115
- self.sliding_window)
116
- input_metadata.attn_bias = attn_bias
117
- else:
118
- input_metadata.attn_bias = _make_alibi_bias(
119
- self.alibi_slopes, self.num_kv_heads, batch_size,
120
- seq_len, query.dtype)
121
-
121
if self.use_ref_attention:
123
- output = _ref_masked_attention(
124
- query,
125
- key,
126
- value,
127
- self.num_heads,
128
- self.num_kv_heads,
129
- self.head_size,
130
- self.scale,
131
- )
122
+ print("ref attention used.")
123
+ output = torch.empty_like(query)
124
+ start = 0
125
+ for _, prompt_len in enumerate(input_metadata.prompt_lens):
126
+ end = start + prompt_len
127
+ out = _ref_masked_attention(
128
+ query[None, start:end],
129
+ key[None, start:end],
130
+ value[None, start:end],
131
+ self.num_heads,
132
+ self.num_kv_heads,
133
+ self.head_size,
134
+ self.scale,
135
+ )
136
+ # TODO(woosuk): Unnecessary copy. Optimize.
137
+ output[start:end].copy_(out)
138
+ start += prompt_len
139
+
140
# Using view got RuntimeError: view size is not compatible
141
# with input tensor's size and stride (at least one
142
# dimension spans across two contiguous subspaces).
143
# Use reshape instead.
136
- return output.reshape(batch_size, seq_len, hidden_size)
137
-
138
- # TODO(woosuk): Too many view operations. Let's try to reduce
139
- # them in the future for code readability.
140
- if self.alibi_slopes is None:
141
- query = query.unsqueeze(0)
142
- key = key.unsqueeze(0)
143
- value = value.unsqueeze(0)
144
- else:
145
- query = query.unflatten(0, (batch_size, seq_len))
146
- key = key.unflatten(0, (batch_size, seq_len))
147
- value = value.unflatten(0, (batch_size, seq_len))
148
-
149
- out = xops.memory_efficient_attention_forward(
150
- query,
151
- key,
152
- value,
153
- attn_bias=input_metadata.attn_bias,
154
- p=0.0,
155
- scale=self.scale,
156
- op=xops.fmha.MemoryEfficientAttentionFlashAttentionOp[0] if
157
- (is_hip()) else None,
158
- )
159
- output = out.view_as(query)
144
+ return output.reshape(num_tokens, hidden_size)
145
146
+ output = self._run_memory_efficient_xformer_forward(
147
+ query, key, value, input_metadata)
148
else:
149
# prefix-enabled attention
150
output = PagedAttentionImpl.forward_prefix(
@@ -182,41 +169,117 @@ class XFormersBackend:
169
)
170
171
# Reshape the output tensor.
185
- return output.view(batch_size, seq_len, hidden_size)
172
+ return output.view(-1, self.num_heads * self.head_size)
173
+
174
+ def _run_memory_efficient_xformer_forward(
175
+ self,
176
+ query: torch.Tensor,
177
+ key: torch.Tensor,
178
+ value: torch.Tensor,
179
+ input_metadata: InputMetadata,
180
+ ) -> torch.Tensor:
181
+ """Attention for 1D query of multiple prompts. Multiple prompt
182
+ tokens are flattened in to `query` input.
183
+
184
+ Args:
185
+ output: shape = [num_prompt_tokens, num_heads, head_size]
186
+ query: shape = [num_prompt_tokens, num_heads, head_size]
187
+ key: shape = [num_prompt_tokens, num_kv_heads, head_size]
188
+ value: shape = [num_prompt_tokens, num_kv_heads, head_size]
189
+ input_metadata: metadata for paged attention.
190
+ """
191
+ # Set attention bias if not provided. This typically happens at
192
+ # the very attention layer of every iteration.
193
+ # FIXME(woosuk): This is a hack.
194
+ if input_metadata.attn_bias is None:
195
+ if self.alibi_slopes is None:
196
+ attn_bias = BlockDiagonalCausalMask.from_seqlens(
197
+ input_metadata.prompt_lens)
198
+ if self.sliding_window is not None:
199
+ attn_bias = attn_bias.make_local_attention(
200
+ self.sliding_window)
201
+ input_metadata.attn_bias = [attn_bias]
202
+ else:
203
+ input_metadata.attn_bias = _make_alibi_bias(
204
+ self.alibi_slopes, self.num_kv_heads, query.dtype,
205
+ input_metadata)
206
+
207
+ op = xops.fmha.MemoryEfficientAttentionFlashAttentionOp[0] if (
208
+ is_hip()) else None
209
+ # No alibi slopes.
210
+ # TODO(woosuk): Too many view operations. Let's try to reduce
211
+ # them in the future for code readability.
212
+ if self.alibi_slopes is None:
213
+ query = query.unsqueeze(0)
214
+ key = key.unsqueeze(0)
215
+ value = value.unsqueeze(0)
216
+ out = xops.memory_efficient_attention_forward(
217
+ query,
218
+ key,
219
+ value,
220
+ attn_bias=input_metadata.attn_bias[0],
221
+ p=0.0,
222
+ scale=self.scale,
223
+ op=op)
224
+
225
+ return out.view_as(query)
226
+
227
+ # Attention with alibi slopes.
228
+ # FIXME(woosuk): Because xformers does not support dynamic sequence
229
+ # lengths with custom attention bias, we process each prompt one by
230
+ # one. This is inefficient, especially when we have many short prompts.
231
+ output = torch.empty_like(query)
232
+ start = 0
233
+ for i, prompt_len in enumerate(input_metadata.prompt_lens):
234
+ end = start + prompt_len
235
+ out = xops.memory_efficient_attention_forward(
236
+ query[None, start:end],
237
+ key[None, start:end],
238
+ value[None, start:end],
239
+ attn_bias=input_metadata.attn_bias[i],
240
+ p=0.0,
241
+ scale=self.scale,
242
+ op=op)
243
+ # TODO(woosuk): Unnecessary copy. Optimize.
244
+ output[start:end].copy_(out.squeeze(0))
245
+ start += prompt_len
246
+ return output
247
248
249
def _make_alibi_bias(
250
alibi_slopes: torch.Tensor,
251
num_kv_heads: int,
191
- batch_size: int,
192
- seq_len: int,
252
dtype: torch.dtype,
253
+ input_metadata: InputMetadata,
254
) -> LowerTriangularMaskWithTensorBias:
195
- bias = torch.arange(seq_len, dtype=dtype)
196
- # NOTE(zhuohan): HF uses
197
- # `bias = bias[None, :].repeat(prompt_len, 1)`
198
- # here. We find that both biases give the same results, but
199
- # the bias below more accurately follows the original ALiBi
200
- # paper.
201
- bias = bias[None, :] - bias[:, None]
202
-
203
- # When using custom attention bias, xformers requires the bias to
204
- # be sliced from a tensor whose length is a multiple of 8.
205
- padded_len = (seq_len + 7) // 8 * 8
206
- num_heads = alibi_slopes.shape[0]
207
- bias = torch.empty(
208
- batch_size,
209
- num_heads,
210
- seq_len,
211
- padded_len,
212
- device=alibi_slopes.device,
213
- dtype=dtype,
214
- )[:, :, :, :seq_len].copy_(bias)
215
- bias.mul_(alibi_slopes[:, None, None])
216
- if num_heads != num_kv_heads:
217
- bias = bias.unflatten(1, (num_kv_heads, num_heads // num_kv_heads))
218
- attn_bias = LowerTriangularMaskWithTensorBias(bias)
219
- return attn_bias
255
+ attn_biases = []
256
+ for prompt_len in input_metadata.prompt_lens:
257
+ bias = torch.arange(prompt_len, dtype=dtype)
258
+ # NOTE(zhuohan): HF uses
259
+ # `bias = bias[None, :].repeat(prompt_len, 1)`
260
+ # here. We find that both biases give the same results, but
261
+ # the bias below more accurately follows the original ALiBi
262
+ # paper.
263
+ # Calculate a matrix where each element represents ith element- jth
264
+ # element.
265
+ bias = bias[None, :] - bias[:, None]
266
+
267
+ padded_len = (prompt_len + 7) // 8 * 8
268
+ num_heads = alibi_slopes.shape[0]
269
+ bias = torch.empty(
270
+ 1, # batch size
271
+ num_heads,
272
+ prompt_len,
273
+ padded_len,
274
+ device=alibi_slopes.device,
275
+ dtype=dtype,
276
+ )[:, :, :, :prompt_len].copy_(bias)
277
+ bias.mul_(alibi_slopes[:, None, None])
278
+ if num_heads != num_kv_heads:
279
+ bias = bias.unflatten(1, (num_kv_heads, num_heads // num_kv_heads))
280
+ attn_biases.append(LowerTriangularMaskWithTensorBias(bias))
281
+
282
+ return attn_biases
283
284
285
def _check_use_ref_attention() -> bool:
@@ -239,7 +302,6 @@ def _ref_masked_attention(
302
query = query.view(-1, num_heads, head_size)
303
key = key.view(-1, num_kv_heads, head_size)
304
value = value.view(-1, num_kv_heads, head_size)
242
-
305
seq_len, _, _ = query.shape
306
attn_mask = torch.triu(torch.ones(seq_len,
307
seq_len,
vllm/model_executor/layers/attention/ops/paged_attn.py
+5
-4
@@ -128,11 +128,12 @@ class PagedAttentionImpl:
128
output,
129
key_cache,
130
value_cache,
131
- input_metadata.block_tables, # [BS, max_block_per_request]
132
- input_metadata.start_loc,
133
- input_metadata.prompt_lens,
131
+ input_metadata.block_tables,
132
+ # subquery_start_loc is (batch_size + 1,)
133
+ input_metadata.subquery_start_loc[:-1],
134
+ input_metadata.prompt_lens_tensor,
135
input_metadata.context_lens,
135
- input_metadata.max_seq_len,
136
+ input_metadata.max_subquery_len,
137
alibi_slopes,
138
)
139
return output
vllm/model_executor/layers/sampler.py
-1
@@ -128,7 +128,6 @@ def _prune_hidden_states(
128
hidden_states: torch.Tensor,
129
sampling_metadata: SamplingMetadata,
130
) -> torch.Tensor:
131
- hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
131
return hidden_states.index_select(0,
132
sampling_metadata.selected_token_indices)
133
vllm/worker/model_runner.py
+144
-95
@@ -28,9 +28,12 @@ logger = init_logger(__name__)
28
KVCache = Tuple[torch.Tensor, torch.Tensor]
29
_PAD_SLOT_ID = -1
30
LORA_WARMUP_RANK = 8
31
-# Capture graphs for batch size 1, 2, 4, 8, 16, 24, 32, 40, ..., 256.
31
+_BATCH_SIZE_ALIGNMENT = 8
32
+# Capture graphs for token size 1, 2, 4, 8, 16, 24, 32, 40, ..., 256.
33
# NOTE: _get_graph_batch_size needs to be updated if this list is changed.
33
-_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [8 * i for i in range(1, 33)]
34
+_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
35
+ _BATCH_SIZE_ALIGNMENT * i for i in range(1, 33)
36
+]
37
38
39
class ModelRunner:
@@ -107,8 +110,7 @@ class ModelRunner:
110
), "Model does not have embedding_padding_modules"
111
self.lora_manager = LRUCacheWorkerLoRAManager(
112
self.scheduler_config.max_num_seqs,
110
- self.scheduler_config.max_num_batched_tokens +
111
- self.scheduler_config.max_paddings, self.vocab_size,
113
+ self.scheduler_config.max_num_batched_tokens, self.vocab_size,
114
self.lora_config, self.device, self.model.embedding_modules,
115
self.model.embedding_padding_modules)
116
self.model = self.lora_manager.create_lora_manager(self.model)
@@ -116,10 +118,13 @@ class ModelRunner:
118
def set_block_size(self, block_size: int) -> None:
119
self.block_size = block_size
120
119
- max_num_blocks = (self.max_context_len_to_capture + block_size -
120
- 1) // block_size
121
self.graph_block_tables = np.zeros(
122
- (max(_BATCH_SIZES_TO_CAPTURE), max_num_blocks), dtype=np.int32)
122
+ (max(_BATCH_SIZES_TO_CAPTURE), self.get_max_block_per_batch()),
123
+ dtype=np.int32)
124
+
125
+ def get_max_block_per_batch(self) -> int:
126
+ block_size = self.block_size
127
+ return (self.max_context_len_to_capture + block_size - 1) // block_size
128
129
def _prepare_prompt(
130
self,
@@ -127,9 +132,9 @@ class ModelRunner:
132
) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, List[int], List[int],
133
List[int], List[int], Set[LoRARequest]]:
134
assert len(seq_group_metadata_list) > 0
130
- input_tokens: List[List[int]] = []
131
- input_positions: List[List[int]] = []
132
- slot_mapping: List[List[int]] = []
135
+ input_tokens: List[int] = []
136
+ input_positions: List[int] = []
137
+ slot_mapping: List[int] = []
138
lora_index_mapping: List[int] = []
139
lora_prompt_mapping: List[int] = []
140
lora_requests: Set[LoRARequest] = set()
@@ -158,16 +163,18 @@ class ModelRunner:
163
computed_len = len(computed_block_nums) * self.block_size
164
prompt_tokens = prompt_tokens[computed_len:]
165
prefix_block_tables.append(computed_block_nums)
166
+ context_len = computed_len
167
else:
168
prefix_block_tables.append([])
169
+ context_len = 0
170
# actual prompt lens
164
- context_lens.append(computed_len)
171
+ context_lens.append(context_len)
172
subquery_lens.append(prompt_len - computed_len)
173
167
- input_tokens.append(prompt_tokens)
174
+ input_tokens.extend(prompt_tokens)
175
# NOTE(woosuk): Here we assume that the first token in the prompt
176
# is always the first token in the sequence.
170
- input_positions.append(
177
+ input_positions.extend(
178
list(range(computed_len, computed_len + len(prompt_tokens))))
179
180
lora_id = seq_group_metadata.lora_int_id
@@ -175,7 +182,7 @@ class ModelRunner:
182
if lora_id > 0:
183
lora_requests.add(seq_group_metadata.lora_request)
184
178
- lora_index_mapping.append([lora_id] * (prompt_len - computed_len))
185
+ lora_index_mapping += [lora_id] * (prompt_len - computed_len)
186
lora_prompt_mapping.extend(
187
[lora_id] *
188
(prompt_len - computed_len
@@ -184,11 +191,10 @@ class ModelRunner:
191
if seq_group_metadata.block_tables is None:
192
# During memory profiling, the block tables are not initialized
193
# yet. In this case, we just use a dummy slot mapping.
187
- slot_mapping.append([_PAD_SLOT_ID] * prompt_len)
194
+ slot_mapping.extend([_PAD_SLOT_ID] * prompt_len)
195
continue
196
197
# Compute the slot mapping.
191
- slot_mapping.append([])
198
block_table = seq_group_metadata.block_tables[seq_id]
199
# Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
200
# where start_idx is max(0, prompt_len - sliding_window).
@@ -203,35 +209,30 @@ class ModelRunner:
209
start_idx = max(0, prompt_len - self.sliding_window)
210
for i in range(computed_len, prompt_len):
211
if i < start_idx:
206
- slot_mapping[-1].append(_PAD_SLOT_ID)
212
+ slot_mapping.append(_PAD_SLOT_ID)
213
continue
214
215
block_number = block_table[i // self.block_size]
216
block_offset = i % self.block_size
217
slot = block_number * self.block_size + block_offset
212
- slot_mapping[-1].append(slot)
213
-
214
- max_prompt_len = max(subquery_lens)
215
- assert max_prompt_len > 0
216
- input_tokens = _make_tensor_with_pad(input_tokens,
217
- max_prompt_len,
218
- pad=0,
219
- dtype=torch.long,
220
- device=self.device)
221
- input_positions = _make_tensor_with_pad(input_positions,
222
- max_prompt_len,
223
- pad=0,
224
- dtype=torch.long,
225
- device=self.device)
226
- slot_mapping = _make_tensor_with_pad(slot_mapping,
227
- max_prompt_len,
228
- pad=_PAD_SLOT_ID,
229
- dtype=torch.long,
230
- device=self.device)
231
- lora_index_mapping = [
232
- _pad_to_max(mapping, max_prompt_len, pad=0)
233
- for mapping in lora_index_mapping
234
- ]
218
+ slot_mapping.append(slot)
219
+
220
+ max_subquery_len = max(subquery_lens)
221
+ max_seq_len = max(prompt_lens)
222
+ num_prompt_tokens = len(input_tokens)
223
+ assert max_subquery_len > 0
224
+
225
+ input_tokens = torch.tensor(input_tokens,
226
+ dtype=torch.long,
227
+ device=self.device)
228
+ input_positions = torch.tensor(input_positions,
229
+ dtype=torch.long,
230
+ device=self.device)
231
+ slot_mapping = torch.tensor(slot_mapping,
232
+ dtype=torch.long,
233
+ device=self.device)
234
+ lora_index_mapping = lora_index_mapping
235
+
236
context_lens_tensor = torch.tensor(context_lens,
237
dtype=torch.int,
238
device=self.device)
@@ -244,22 +245,45 @@ class ModelRunner:
245
dtype=torch.int,
246
device=self.device,
247
)
247
- start_loc_tensor = torch.arange(0,
248
- len(prompt_lens) * max_prompt_len,
249
- max_prompt_len,
250
- dtype=torch.long,
251
- device=self.device)
248
+
249
+ # Query length can be shorter than key (i.e., prompt) when prefill
250
+ # is chunked or prefix cached.
251
+ subquery_lens_tensor = torch.tensor(subquery_lens,
252
+ dtype=torch.long,
253
+ device=self.device)
254
+ subquery_start_loc = torch.zeros(subquery_lens_tensor.shape[0] + 1,
255
+ dtype=torch.int32,
256
+ device=self.device)
257
+
258
prompt_lens_tensor = torch.tensor(prompt_lens,
259
dtype=torch.long,
260
device=self.device)
261
+ seq_start_loc = torch.zeros(prompt_lens_tensor.shape[0] + 1,
262
+ dtype=torch.int32,
263
+ device=self.device)
264
+
265
+ torch.cumsum(subquery_lens_tensor,
266
+ dim=0,
267
+ dtype=subquery_start_loc.dtype,
268
+ out=subquery_start_loc[1:])
269
+
270
+ torch.cumsum(prompt_lens_tensor,
271
+ dim=0,
272
+ dtype=seq_start_loc.dtype,
273
+ out=seq_start_loc[1:])
274
275
input_metadata = InputMetadata(
276
is_prompt=True,
277
slot_mapping=slot_mapping,
259
- prompt_lens=prompt_lens_tensor,
260
- max_seq_len=max_prompt_len,
261
- start_loc=start_loc_tensor,
278
+ prompt_lens=prompt_lens,
279
+ prompt_lens_tensor=prompt_lens_tensor,
280
+ num_prompt_tokens=num_prompt_tokens,
281
+ num_generation_tokens=0,
282
+ max_subquery_len=max_subquery_len,
283
max_context_len=None,
284
+ max_seq_len=max_seq_len,
285
+ subquery_start_loc=subquery_start_loc,
286
+ seq_start_loc=seq_start_loc,
287
context_lens=context_lens_tensor,
288
block_tables=block_tables,
289
use_cuda_graph=False,
@@ -275,9 +299,9 @@ class ModelRunner:
299
) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, List[int], List[int],
300
Set[LoRARequest]]:
301
assert len(seq_group_metadata_list) > 0
278
- input_tokens: List[List[int]] = []
279
- input_positions: List[List[int]] = []
280
- slot_mapping: List[List[int]] = []
302
+ input_tokens: List[int] = []
303
+ input_positions: List[int] = []
304
+ slot_mapping: List[int] = []
305
context_lens: List[int] = []
306
block_tables: List[List[int]] = []
307
lora_index_mapping: List[int] = []
@@ -296,11 +320,11 @@ class ModelRunner:
320
for seq_id in seq_ids:
321
seq_data = seq_group_metadata.seq_data[seq_id]
322
generation_token = seq_data.get_last_token_id()
299
- input_tokens.append([generation_token])
323
+ input_tokens.append(generation_token)
324
325
seq_len = seq_data.get_len()
326
position = seq_len - 1
303
- input_positions.append([position])
327
+ input_positions.append(position)
328
329
context_len = seq_len if self.sliding_window is None else min(
330
seq_len, self.sliding_window)
@@ -310,8 +334,8 @@ class ModelRunner:
334
block_number = block_table[position // self.block_size]
335
block_offset = position % self.block_size
336
slot = block_number * self.block_size + block_offset
313
- slot_mapping.append([slot])
314
- lora_index_mapping.append([lora_id])
337
+ slot_mapping.append(slot)
338
+ lora_index_mapping.append(lora_id)
339
lora_prompt_mapping.append(lora_id)
340
341
if self.sliding_window is not None:
@@ -320,6 +344,9 @@ class ModelRunner:
344
block_table = block_table[-sliding_window_blocks:]
345
block_tables.append(block_table)
346
347
+ # vLLM uses cuda graph only for decoding requests.
348
+ # See `capture_model` API for more details.
349
+ # For decoding requests, batch_size == input_tokens.
350
batch_size = len(input_tokens)
351
max_context_len = max(context_lens)
352
use_captured_graph = (
@@ -327,38 +354,37 @@ class ModelRunner:
354
and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
355
and max_context_len <= self.max_context_len_to_capture)
356
if use_captured_graph:
330
- # Pad the input tokens, positions, and slot mapping to match the
331
- # batch size of the captured graph.
357
graph_batch_size = _get_graph_batch_size(batch_size)
358
assert graph_batch_size >= batch_size
359
for _ in range(graph_batch_size - batch_size):
335
- input_tokens.append([])
336
- input_positions.append([])
337
- slot_mapping.append([])
360
+ input_tokens.append(0)
361
+ input_positions.append(0)
362
+ slot_mapping.append(_PAD_SLOT_ID)
363
context_lens.append(1)
364
block_tables.append([])
365
+ lora_index_mapping.append(0)
366
batch_size = graph_batch_size
367
342
- input_tokens = _make_tensor_with_pad(input_tokens,
343
- max_len=1,
344
- pad=0,
345
- dtype=torch.long,
346
- device=self.device)
347
- input_positions = _make_tensor_with_pad(input_positions,
348
- max_len=1,
349
- pad=0,
350
- dtype=torch.long,
351
- device=self.device)
352
- slot_mapping = _make_tensor_with_pad(slot_mapping,
353
- max_len=1,
354
- pad=_PAD_SLOT_ID,
355
- dtype=torch.long,
356
- device=self.device)
368
+ input_tokens = torch.tensor(input_tokens,
369
+ dtype=torch.long,
370
+ device=self.device)
371
+ input_positions = torch.tensor(input_positions,
372
+ dtype=torch.long,
373
+ device=self.device)
374
+ slot_mapping = torch.tensor(slot_mapping,
375
+ dtype=torch.long,
376
+ device=self.device)
377
context_lens = torch.tensor(context_lens,
378
dtype=torch.int,
379
device=self.device)
380
381
if use_captured_graph:
382
+ # When using cuda-graph all these tensors should be
383
+ # padded.
384
+ assert context_lens.shape[0] == input_tokens.shape[0]
385
+ assert context_lens.shape[0] == input_positions.shape[0]
386
+ assert context_lens.shape[0] == slot_mapping.shape[0]
387
+
388
# The shape of graph_block_tables is
389
# [max batch size, max context len // block size].
390
input_block_tables = self.graph_block_tables[:batch_size]
@@ -377,17 +403,18 @@ class ModelRunner:
403
device=self.device,
404
)
405
380
- lora_index_mapping = [
381
- _pad_to_max(mapping, 1, pad=0) for mapping in lora_index_mapping
382
- ]
383
-
406
input_metadata = InputMetadata(
407
is_prompt=False,
408
slot_mapping=slot_mapping,
409
prompt_lens=None,
388
- max_seq_len=None,
389
- start_loc=None,
410
+ prompt_lens_tensor=None,
411
+ num_prompt_tokens=0,
412
+ num_generation_tokens=len(input_tokens),
413
+ max_subquery_len=None,
414
max_context_len=max_context_len,
415
+ max_seq_len=None,
416
+ subquery_start_loc=None,
417
+ seq_start_loc=None,
418
context_lens=context_lens,
419
block_tables=block_tables,
420
use_cuda_graph=use_captured_graph,
@@ -411,7 +438,6 @@ class ModelRunner:
438
categorized_sampled_token_indices_start_idx = 0
439
pin_memory = not self.in_wsl and not self.device_config.is_neuron
440
414
- max_subquery_len = max(subquery_lens) if subquery_lens else 1
441
for i, seq_group_metadata in enumerate(seq_group_metadata_list):
442
seq_ids = list(seq_group_metadata.seq_data.keys())
443
sampling_params = seq_group_metadata.sampling_params
@@ -439,7 +465,7 @@ class ModelRunner:
465
selected_token_start_idx + subquery_len - 1))
466
selected_token_indices.append(selected_token_start_idx +
467
subquery_len - 1)
442
- selected_token_start_idx += max_subquery_len
468
+ selected_token_start_idx += subquery_len
469
470
if sampling_params.seed is not None:
471
seq_group_metadata.state.generator = torch.Generator(
@@ -521,11 +547,8 @@ class ModelRunner:
547
subquery_lens)
548
549
if self.lora_config:
524
- flat_lora_index_mapping = [
525
- item for sublist in lora_index_mapping for item in sublist
526
- ]
550
lora_mapping = LoRAMapping(
528
- flat_lora_index_mapping,
551
+ lora_index_mapping,
552
lora_prompt_mapping,
553
)
554
else:
@@ -679,6 +702,18 @@ class ModelRunner:
702
703
@torch.inference_mode()
704
def capture_model(self, kv_caches: List[KVCache]) -> None:
705
+ """Cuda graph capture a model.
706
+
707
+ Note that CUDA graph's performance gain is negligible if number
708
+ of batched tokens are larger than 200. And since CUDA graph
709
+ requires fixed sized tensors, supporting large/variable batch
710
+ size requires high GPU memory overhead. Thus, vLLM only captures
711
+ decoding requests. Mixed batch (chunked prefill + decoding) or
712
+ prefill requests are not captured.
713
+
714
+ Since it is used for decoding-only, it assumes there's only 1 token
715
+ per sequence in the batch.
716
+ """
717
# NOTE(woosuk): This is a hack to ensure that the NCCL backend is never
718
# deleted before the CUDA graphs.
719
self.cupy_nccl_backend = cupy_utils.get_nccl_backend()
@@ -697,10 +732,9 @@ class ModelRunner:
732
733
# Prepare dummy inputs. These will be reused for all batch sizes.
734
max_batch_size = max(_BATCH_SIZES_TO_CAPTURE)
700
- input_tokens = torch.zeros(max_batch_size, 1, dtype=torch.long).cuda()
701
- input_positions = torch.zeros(max_batch_size, 1,
702
- dtype=torch.long).cuda()
703
- slot_mapping = torch.empty(max_batch_size, 1, dtype=torch.long).cuda()
735
+ input_tokens = torch.zeros(max_batch_size, dtype=torch.long).cuda()
736
+ input_positions = torch.zeros(max_batch_size, dtype=torch.long).cuda()
737
+ slot_mapping = torch.empty(max_batch_size, dtype=torch.long).cuda()
738
slot_mapping.fill_(_PAD_SLOT_ID)
739
context_lens = torch.ones(max_batch_size, dtype=torch.int32).cuda()
740
block_tables = torch.from_numpy(self.graph_block_tables).cuda()
@@ -726,9 +760,14 @@ class ModelRunner:
760
is_prompt=False,
761
slot_mapping=slot_mapping[:batch_size],
762
prompt_lens=None,
729
- max_seq_len=None,
730
- start_loc=None,
763
+ prompt_lens_tensor=None,
764
+ num_prompt_tokens=0,
765
+ num_generation_tokens=batch_size,
766
+ max_subquery_len=None,
767
max_context_len=self.max_context_len_to_capture,
768
+ max_seq_len=None,
769
+ subquery_start_loc=None,
770
+ seq_start_loc=None,
771
context_lens=context_lens[:batch_size],
772
block_tables=block_tables[:batch_size],
773
use_cuda_graph=True,
@@ -845,7 +884,6 @@ class CUDAGraphRunner:
884
non_blocking=True)
885
self.input_buffers["block_tables"].copy_(input_metadata.block_tables,
886
non_blocking=True)
848
-
887
# Run the graph.
888
self.graph.replay()
889
@@ -877,17 +915,28 @@ def _make_tensor_with_pad(
915
dtype: torch.dtype,
916
device: Optional[Union[str, torch.device]],
917
) -> torch.Tensor:
918
+ """Make a padded tensor of a 2D inputs.
919
+
920
+ The padding is applied to the end of each inner list until it reaches
921
+ `max_len`.
922
+ """
923
padded_x = [_pad_to_max(x_i, max_len, pad) for x_i in x]
924
return torch.tensor(padded_x, dtype=dtype, device=device)
925
926
927
def _get_graph_batch_size(batch_size: int) -> int:
928
+ """Returns the padded batch size given actual batch size.
929
+
930
+ Batch sizes are 1, 2, 4, _BATCH_SIZE_ALIGNMENT,
931
+ 2*_BATCH_SIZE_ALIGNMENT, 3*_BATCH_SIZE_ALIGNMENT...
932
+ """
933
if batch_size <= 2:
934
return batch_size
935
elif batch_size <= 4:
936
return 4
937
else:
890
- return (batch_size + 7) // 8 * 8
938
+ return ((batch_size + _BATCH_SIZE_ALIGNMENT - 1) //
939
+ _BATCH_SIZE_ALIGNMENT * _BATCH_SIZE_ALIGNMENT)
940
941
942
def _async_h2d(