@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6e435de766
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[1/n][Chunked Prefill] Refactor input query shapes (#3236)

SangBin Cho committed Mar 21, 2024 at 06:46 UTC 6e435de766c7749b214b637ac58570a221006c95
18 files changed +575 -259
.buildkite/test-pipeline.yaml
+2 -2
@@ -47,7 +47,7 @@ steps:
47 - pytest -v -s prefix_caching
48
49 - label: Samplers Test
50 - command: pytest -v -s samplers --forked
50 + command: pytest -v -s samplers
51
52 - label: Worker Test
53 command: pytest -v -s worker
@@ -56,7 +56,7 @@ steps:
56 command: pytest -v -s spec_decode
57
58 - label: LoRA Test %N
59 - command: pytest -v -s lora --forked --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
59 + command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
60 parallelism: 4
61
62 - label: Metrics Test
tests/basic_correctness/test_basic_correctness.py
+3 -1
@@ -13,6 +13,7 @@ MODELS = [
13 @pytest.mark.parametrize("model", MODELS)
14 @pytest.mark.parametrize("dtype", ["half"])
15 @pytest.mark.parametrize("max_tokens", [5])
16 +@pytest.mark.parametrize("enforce_eager", [False, True])
17 def test_models(
18 hf_runner,
19 vllm_runner,
@@ -20,12 +21,13 @@ def test_models(
21 model: str,
22 dtype: str,
23 max_tokens: int,
24 + enforce_eager: bool,
25 ) -> None:
26 hf_model = hf_runner(model, dtype=dtype)
27 hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
28 del hf_model
29
28 - vllm_model = vllm_runner(model, dtype=dtype)
30 + vllm_model = vllm_runner(model, dtype=dtype, enforce_eager=enforce_eager)
31 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
32 del vllm_model
33
tests/core/test_scheduler.py
+9 -9
@@ -10,7 +10,7 @@ from .utils import create_dummy_prompt
10
11 def test_scheduler_add_seq_group():
12 block_size = 4
13 - scheduler_config = SchedulerConfig(100, 64, 1, 256)
13 + scheduler_config = SchedulerConfig(100, 64, 1)
14 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
15 cache_config.num_cpu_blocks = 4
16 cache_config.num_gpu_blocks = 4
@@ -26,7 +26,7 @@ def test_scheduler_add_seq_group():
26
27 def test_scheduler_abort_seq_group():
28 block_size = 4
29 - scheduler_config = SchedulerConfig(100, 64, 1, 256)
29 + scheduler_config = SchedulerConfig(100, 64, 1)
30 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
31 cache_config.num_cpu_blocks = 4
32 cache_config.num_gpu_blocks = 4
@@ -50,7 +50,7 @@ def test_scheduler_schedule_simple():
50 block_size = 4
51 num_seq_group = 4
52 max_model_len = 16
53 - scheduler_config = SchedulerConfig(64, num_seq_group, max_model_len, 256)
53 + scheduler_config = SchedulerConfig(64, num_seq_group, max_model_len)
54 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
55 cache_config.num_cpu_blocks = 8
56 cache_config.num_gpu_blocks = 8
@@ -64,10 +64,10 @@ def test_scheduler_schedule_simple():
64 running.append(seq_group)
65
66 # Schedule seq groups prompts.
67 + num_tokens = block_size * num_seq_group
68 seq_group_meta, out = scheduler.schedule()
69 assert set(out.scheduled_seq_groups) == set(running)
69 - assert out.num_batched_tokens == num_seq_group * seq_group.get_seqs(
70 - )[0].get_len()
70 + assert out.num_batched_tokens == num_tokens
71 assert (not out.blocks_to_copy and not out.blocks_to_swap_in
72 and not out.blocks_to_swap_out)
73 assert len(seq_group_meta) == num_seq_group
@@ -84,7 +84,7 @@ def test_scheduler_schedule_simple():
84 def test_scheduler_schedule_preempt_abort():
85 block_size = 4
86 max_model_len = 16
87 - scheduler_config = SchedulerConfig(64, 2, max_model_len, 256)
87 + scheduler_config = SchedulerConfig(64, 2, max_model_len)
88 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
89 cache_config.num_cpu_blocks = 2
90 cache_config.num_gpu_blocks = 2
@@ -99,7 +99,7 @@ def test_scheduler_schedule_preempt_abort():
99 # Schedule seq groups prompts.
100 seq_group_meta, out = scheduler.schedule()
101 assert out.scheduled_seq_groups == [seq_group_a, seq_group_b]
102 - assert out.num_batched_tokens == seq_group_a.get_seqs()[0].get_len() * 2
102 + assert out.num_batched_tokens == block_size * 2 # seq_a and seq_b
103 assert (not out.blocks_to_copy and not out.blocks_to_swap_in
104 and not out.blocks_to_swap_out)
105 assert len(seq_group_meta) == 2
@@ -124,7 +124,7 @@ def test_scheduler_schedule_preempt_abort():
124 scheduler.abort_seq_group("1")
125 seq_group_meta, out = scheduler.schedule()
126 assert out.scheduled_seq_groups == [seq_group_b]
127 - assert out.num_batched_tokens == seq_group_b.get_seqs()[0].get_len()
127 + assert out.num_batched_tokens == 5 # 4 prompt + 1 generation.
128 assert (not out.blocks_to_copy and not out.blocks_to_swap_in
129 and not out.blocks_to_swap_out)
130 assert len(seq_group_meta) == 1
@@ -136,7 +136,7 @@ def test_scheduler_max_seqs():
136 num_seq_group = 4
137 max_seq_group = 2
138 max_model_len = 16
139 - scheduler_config = SchedulerConfig(64, max_seq_group, max_model_len, 256)
139 + scheduler_config = SchedulerConfig(64, max_seq_group, max_model_len)
140 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
141 cache_config.num_cpu_blocks = 8
142 cache_config.num_gpu_blocks = 8
tests/lora/test_worker.py
+1 -1
@@ -25,7 +25,7 @@ def test_worker_apply_lora(sql_lora_files):
25 revision=None,
26 ),
27 parallel_config=ParallelConfig(1, 1, False),
28 - scheduler_config=SchedulerConfig(32, 32, 32, 256),
28 + scheduler_config=SchedulerConfig(32, 32, 32),
29 device_config=DeviceConfig("cuda"),
30 local_rank=0,
31 rank=0,
tests/spec_decode/test_multi_step_worker.py
+2 -2
@@ -92,8 +92,8 @@ def test_same_output_for_single_step():
92 num_gpu_blocks,
93 seed,
94 )
95 - multi_step_worker.model_runner = worker.model_runner
96 - multi_step_worker.cache_engine = worker.cache_engine
95 + # multi_step_worker.model_runner = worker.model_runner
96 + # multi_step_worker.cache_engine = worker.cache_engine
97
98 num_steps = 1
99
tests/worker/test_model_runner.py
+153 -8
@@ -1,8 +1,13 @@
1 import random
2 import torch
3
4 +from vllm.config import ModelConfig
5 from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
5 -from vllm.worker.model_runner import ModelRunner
6 +from vllm.worker.model_runner import ModelRunner, _BATCH_SIZE_ALIGNMENT
7 +
8 +
9 +def get_aligned_size(batch_size: int, alignment: int):
10 + return ((batch_size + alignment - 1) // alignment * alignment)
11
12
13 def test_prepare_prompt():
@@ -12,6 +17,7 @@ def test_prepare_prompt():
17 batch_size = random.randint(1, 256)
18 prompt_lens = []
19 seq_group_metadata_list = []
20 + block_tables = {0: [1]}
21 for i in range(batch_size):
22 # make sure all tokens fit into one block
23 prompt_len = i % (model_runner.block_size - 1) + 1
@@ -23,26 +29,165 @@ def test_prepare_prompt():
29 is_prompt=True,
30 seq_data={0: SequenceData(seq_data)},
31 sampling_params=SamplingParams(temperature=0),
26 - block_tables={0: [1]},
32 + block_tables=block_tables,
33 ))
34
35 expected_selected_token_indices = []
36 selected_token_start_idx = 0
31 - max_seq_len = max(prompt_lens)
37 for prompt_len in prompt_lens:
38 expected_selected_token_indices.append(selected_token_start_idx +
39 prompt_len - 1)
35 - selected_token_start_idx += max_seq_len
36 - input_tokens, input_positions, _, return_prompt_lens, _, _, _, _ = (
37 - model_runner._prepare_prompt(seq_group_metadata_list))
40 + selected_token_start_idx += prompt_len
41 + (input_tokens, input_positions, input_metadata, return_prompt_lens, _, _,
42 + _, _) = (model_runner._prepare_prompt(seq_group_metadata_list))
43 assert return_prompt_lens == prompt_lens
44 +
45 + # Verify input metadata is correct for prompts.
46 + device = model_runner.device
47 + assert input_metadata.is_prompt is True
48 + assert torch.allclose(input_metadata.prompt_lens_tensor,
49 + torch.tensor(prompt_lens, device=device))
50 + assert input_metadata.prompt_lens == prompt_lens
51 + assert input_metadata.num_prompt_tokens == sum(prompt_lens)
52 + assert input_metadata.num_generation_tokens == 0
53 + assert input_metadata.max_seq_len == max(prompt_lens)
54 +
55 + # Test subquery start locs.
56 + start_idx = 0
57 + start_loc = [start_idx]
58 + for prompt_len in prompt_lens:
59 + start_idx += prompt_len
60 + start_loc.append(start_idx)
61 + assert torch.allclose(
62 + input_metadata.subquery_start_loc,
63 + torch.tensor(start_loc, dtype=torch.int32, device=device))
64 +
65 + # Test seq start locs. Note that for normal prefill it is
66 + # equivalent to subquery_start_loc.
67 + start_idx = 0
68 + seq_start_loc = [start_idx]
69 + for prompt_len in prompt_lens:
70 + start_idx += prompt_len
71 + seq_start_loc.append(start_idx)
72 +
73 + assert torch.allclose(
74 + input_metadata.seq_start_loc,
75 + torch.tensor(start_loc, dtype=torch.int32, device=device))
76 + assert input_metadata.max_context_len is None
77 + assert torch.allclose(
78 + input_metadata.context_lens,
79 + torch.zeros(input_metadata.context_lens.shape[0],
80 + dtype=torch.int,
81 + device=device))
82 +
83 + expected = torch.tensor([[] for _ in range(len(seq_group_metadata_list))],
84 + dtype=torch.int32,
85 + device=model_runner.device)
86 + assert torch.allclose(input_metadata.block_tables, expected)
87 + # Cuda graph should not be used for prerill.
88 + assert input_metadata.use_cuda_graph is False
89 + assert input_metadata.kv_cache_dtype == "auto"
90 +
91 + assert input_tokens.shape == (sum(prompt_lens), )
92 + assert input_positions.shape == (sum(prompt_lens), )
93 + torch.testing.assert_close(input_tokens, input_positions)
94 +
95 sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
96 prompt_lens,
97 subquery_lens=prompt_lens)
42 - assert input_tokens.shape == (batch_size, max_seq_len)
43 - assert input_positions.shape == (batch_size, max_seq_len)
98 + assert input_tokens.shape == (sum(prompt_lens), )
99 + assert input_positions.shape == (sum(prompt_lens), )
100 + actual = sampling_metadata.selected_token_indices
101 + expected = torch.tensor(expected_selected_token_indices,
102 + device=actual.device,
103 + dtype=actual.dtype)
104 + torch.testing.assert_close(actual, expected)
105 + torch.testing.assert_close(input_tokens, input_positions)
106 +
107 + actual = sampling_metadata.selected_token_indices
108 + expected = torch.tensor(expected_selected_token_indices,
109 + device=actual.device,
110 + dtype=actual.dtype)
111 + torch.testing.assert_close(actual, expected)
112 +
113 +
114 +def test_prepare_decode_cuda_graph():
115 + model_config = ModelConfig(
116 + "facebook/opt-125m",
117 + "facebook/opt-125m",
118 + tokenizer_mode="auto",
119 + trust_remote_code=False,
120 + download_dir=None,
121 + load_format="dummy",
122 + seed=0,
123 + dtype="float16",
124 + revision=None,
125 + enforce_eager=False,
126 + )
127 + model_runner = ModelRunner(model_config, None, None, None, None)
128 + model_runner.set_block_size(16)
129 +
130 + batch_size = random.randint(1, 256)
131 + prompt_lens = []
132 + seq_group_metadata_list = []
133 + for i in range(batch_size):
134 + # make sure all tokens fit into one block
135 + prompt_len = i % (model_runner.block_size - 1) + 1
136 + prompt_lens.append(prompt_len)
137 + seq_data = list(range(prompt_len))
138 + seq_group_metadata_list.append(
139 + SequenceGroupMetadata(
140 + request_id=f"test_{i}",
141 + is_prompt=False,
142 + seq_data={0: SequenceData(seq_data)},
143 + sampling_params=SamplingParams(temperature=0),
144 + block_tables={0: [1]},
145 + ))
146 +
147 + input_tokens, input_positions, input_metadata, _, _, _ = (
148 + model_runner._prepare_decode(seq_group_metadata_list))
149 +
150 + # Verify input metadata is correct for prompts.
151 + device = model_runner.device
152 + assert input_metadata.is_prompt is False
153 + assert input_metadata.prompt_lens is None
154 + assert input_metadata.num_prompt_tokens == 0
155 + assert input_metadata.num_generation_tokens == (get_aligned_size(
156 + len(seq_group_metadata_list), _BATCH_SIZE_ALIGNMENT))
157 + assert input_metadata.max_seq_len is None
158 + assert input_metadata.subquery_start_loc is None
159 + assert input_metadata.seq_start_loc is None
160 + assert input_metadata.max_context_len == max(prompt_lens)
161 + assert torch.allclose(
162 + input_metadata.context_lens[:len(prompt_lens)],
163 + torch.tensor(prompt_lens, dtype=torch.int, device=device))
164 +
165 + # block table's first index corresponds to each batch, meaning in
166 + # decoding it is each token.
167 + assert input_metadata.block_tables.shape[0] == len(input_tokens)
168 + # Block table's second dim correspondsd to each token's block number.
169 + # It is padded up to
170 + assert input_metadata.block_tables.shape[1] == (
171 + model_runner.get_max_block_per_batch())
172 + # Cuda graph should not be used for prerill.
173 + assert input_metadata.use_cuda_graph is True
174 + assert input_metadata.kv_cache_dtype == "auto"
175 +
176 + assert input_tokens.shape == (get_aligned_size(
177 + len(seq_group_metadata_list), _BATCH_SIZE_ALIGNMENT), )
178 + assert input_positions.shape == (get_aligned_size(
179 + len(seq_group_metadata_list), _BATCH_SIZE_ALIGNMENT), )
180 torch.testing.assert_close(input_tokens, input_positions)
181
182 + # Verify Sampling
183 + expected_selected_token_indices = []
184 + selected_token_start_idx = 0
185 + for prompt_len in prompt_lens:
186 + expected_selected_token_indices.append(selected_token_start_idx)
187 + selected_token_start_idx += 1
188 + sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
189 + prompt_lens,
190 + subquery_lens=prompt_lens)
191 actual = sampling_metadata.selected_token_indices
192 expected = torch.tensor(expected_selected_token_indices,
193 device=actual.device,
vllm/config.py
-3
@@ -535,7 +535,6 @@ class SchedulerConfig:
535 iteration.
536 max_model_len: Maximum length of a sequence (including prompt
537 and generated text).
538 - max_paddings: Maximum number of paddings to be added to a batch.
538 """
539
540 def __init__(
@@ -543,7 +542,6 @@ class SchedulerConfig:
542 max_num_batched_tokens: Optional[int],
543 max_num_seqs: int,
544 max_model_len: int,
546 - max_paddings: int,
545 ) -> None:
546 if max_num_batched_tokens is not None:
547 self.max_num_batched_tokens = max_num_batched_tokens
@@ -553,7 +551,6 @@ class SchedulerConfig:
551 self.max_num_batched_tokens = max(max_model_len, 2048)
552 self.max_num_seqs = max_num_seqs
553 self.max_model_len = max_model_len
556 - self.max_paddings = max_paddings
554 self._verify_args()
555
556 def _verify_args(self) -> None:
vllm/core/scheduler.py
+3 -10
@@ -173,12 +173,12 @@ class Scheduler:
173 curr_loras = set(
174 seq_group.lora_int_id
175 for seq_group in self.running) if self.lora_enabled else None
176 - seq_lens: List[int] = []
176
177 # Optimization: We do not sort the waiting queue since the preempted
178 # sequence groups are added to the front and the new sequence groups
179 # are added to the back.
180 leftover_waiting_sequences = deque()
181 + num_batched_tokens = 0
182 while self.waiting:
183 seq_group = self.waiting[0]
184 waiting_seqs = seq_group.get_seqs(
@@ -223,8 +223,7 @@ class Scheduler:
223 continue
224
225 # If the number of batched tokens exceeds the limit, stop.
226 - new_seq_lens = seq_lens + [num_prompt_tokens]
227 - num_batched_tokens = len(new_seq_lens) * max(new_seq_lens)
226 + num_batched_tokens += num_prompt_tokens
227 if (num_batched_tokens >
228 self.scheduler_config.max_num_batched_tokens):
229 break
@@ -236,11 +235,6 @@ class Scheduler:
235 self.scheduler_config.max_num_seqs):
236 break
237
239 - num_paddings = num_batched_tokens - sum(new_seq_lens)
240 - if num_paddings > self.scheduler_config.max_paddings:
241 - break
242 - seq_lens = new_seq_lens
243 -
238 if lora_int_id > 0:
239 curr_loras.add(lora_int_id)
240 self.waiting.popleft()
@@ -255,8 +249,7 @@ class Scheduler:
249 scheduler_outputs = SchedulerOutputs(
250 scheduled_seq_groups=scheduled,
251 prompt_run=True,
258 - num_batched_tokens=len(seq_lens) *
259 - max(seq_lens) if seq_lens else 0,
252 + num_batched_tokens=num_batched_tokens,
253 blocks_to_swap_in=blocks_to_swap_in,
254 blocks_to_swap_out=blocks_to_swap_out,
255 blocks_to_copy=blocks_to_copy,
vllm/engine/arg_utils.py
+1 -7
@@ -31,7 +31,6 @@ class EngineArgs:
31 gpu_memory_utilization: float = 0.90
32 max_num_batched_tokens: Optional[int] = None
33 max_num_seqs: int = 256
34 - max_paddings: int = 256
34 max_logprobs: int = 5 # OpenAI default value
35 disable_log_stats: bool = False
36 revision: Optional[str] = None
@@ -213,10 +212,6 @@ class EngineArgs:
212 type=int,
213 default=EngineArgs.max_num_seqs,
214 help='maximum number of sequences per iteration')
216 - parser.add_argument('--max-paddings',
217 - type=int,
218 - default=EngineArgs.max_paddings,
219 - help='maximum number of paddings in a batch')
215 parser.add_argument(
216 '--max-logprobs',
217 type=int,
@@ -347,8 +342,7 @@ class EngineArgs:
342 ), self.ray_workers_use_nsight)
343 scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
344 self.max_num_seqs,
350 - model_config.max_model_len,
351 - self.max_paddings)
345 + model_config.max_model_len)
346 lora_config = LoRAConfig(
347 max_lora_rank=self.max_lora_rank,
348 max_loras=self.max_loras,
vllm/engine/llm_engine.py
-1
@@ -561,7 +561,6 @@ class LLMEngine:
561 # Log stats.
562 if self.log_stats:
563 self.stat_logger.log(self._get_stats(scheduler_outputs))
564 -
564 return request_outputs
565
566 def step(self) -> List[RequestOutput]:
vllm/model_executor/input_metadata.py
+69 -13
@@ -1,36 +1,92 @@
1 from dataclasses import dataclass, fields
2 -from typing import Optional, Any, Dict
2 +from typing import Optional, List, Any, Dict
3
4 import torch
5 +from xformers.ops.fmha.attn_bias import AttentionBias
6
7
8 @dataclass
9 class InputMetadata:
10 """Metadata for input sequences. Used in PagedAttention.
11
11 - Args:
12 - prompt_lens: Lengths of prompts.
13 - slot_mapping: The address to write the new KV to of each token.
14 - max_context_len: The maximum context length.
15 - context_lens: the length of attention context for each sequence.
16 - block_tables: The block tables. (Seq id -> list of physical block)
17 - kv_cache_dtype: Data type to store kv cache.
12 + NOTE: Any python object stored here is not updated when it is
13 + cuda-graph replayed. If you have values that need to be changed
14 + dynamically, it should be stored in tensor. The tensor has to be
15 + updated from `CUDAGraphRunner.forward` API.
16 """
19 -
17 + # Currently, input sequences can only contain all prompts
18 + # or all decoding. True if all sequences are prompts.
19 is_prompt: bool
20 + # (num_tokens,). The indices of the token slots that input tokens will be
21 + # stored into. E.g., if `slot_mapping` is [35, 2, 17] and the block size
22 + # is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot
23 + # in block 0, and 1st slot in block 1, respectively.
24 slot_mapping: torch.Tensor
22 - prompt_lens: Optional[torch.Tensor]
23 - max_seq_len: Optional[int]
24 - start_loc: Optional[torch.Tensor]
25 + # (batch_size,). The prompt length per sequence. None if it is a decoding.
26 + prompt_lens: Optional[List[int]]
27 + # prompt_lens stored as a tensor.
28 + prompt_lens_tensor: Optional[torch.Tensor]
29 + # The number of prompt tokens. Doesn't include padding.
30 + num_prompt_tokens: int
31 + # The number of generation tokens. Doesn't include padding.
32 + num_generation_tokens: int
33 + """
34 + Definition of context_len, subquery_len, and seqlen.
35 + |---------- N-1 iteration --------|
36 + |---------------- N iteration ---------------------|
37 + |- tokenA -|......................|-- newTokens ---|
38 + |---------- context_len ----------|
39 + |-------------------- seqlen ----------------------|
40 + |- subquery_len -|
41 +
42 + WARNING: context_len has different definition depending on if it is
43 + prefill vs decoding. When it is prefill, it doesn't include new
44 + tokens. When it is for decoding, it includes a new token.
45 + """
46 +
47 + # Maximum subquery length in the batch.
48 + max_subquery_len: Optional[int]
49 + # Maximum context length in the batch.
50 max_context_len: Optional[int]
51 + # FIXME: It is for flash attn.
52 + # Maximum sequence length in the batch.
53 + max_seq_len: Optional[int]
54 + # (batch_size + 1,). The cumulative subquery lengths of the sequences in
55 + # the batch, used to index into subquery. E.g., if the subquery length
56 + # is [4, 6], it is [0, 4, 10].
57 + subquery_start_loc: Optional[torch.Tensor]
58 + # FIXME: It is for flash attn.
59 + # (batch_size + 1,). The cumulative sequence lengths of the sequences in
60 + # the batch, used to index into sequence. E.g., if the sequence length is
61 + # [4, 6], it is [0, 4, 10].
62 + seq_start_loc: Optional[torch.Tensor]
63 + # (batch_size,). The length of context (tokens stored in KV cache) per
64 + # sequence. WARNING: When it is a prefill request, it doesn't include new
65 + # tokens. When it is for decoding, it includes a new token.
66 context_lens: Optional[torch.Tensor]
67 + # (batch_size, max_blocks_per_seq).
68 + # Block addresses per sequence. (Seq id -> list of physical block)
69 + # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks
70 + # in the kv cache. Each block can contain up to block_size tokens.
71 + # 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph
72 + # captured.
73 block_tables: Optional[torch.Tensor]
74 + # Whether or not if cuda graph is enabled.
75 + # Cuda-graph is currently enabled for decoding only.
76 use_cuda_graph: bool
77 kv_cache_dtype: str
78
79 def __post_init__(self):
80 + # Set during the execution of the first attention op.
81 + # It is a list because it is needed to set per prompt
82 + # when alibi slopes is used. It is because of the limitation
83 + # from xformer API.
84 # will not appear in the __repr__ and __init__
33 - self.attn_bias = None
85 + self.attn_bias: Optional[List[AttentionBias]] = None
86 +
87 + # Cuda graph is only used for decoding now.
88 + if self.use_cuda_graph:
89 + assert self.num_prompt_tokens == 0
90
91 def asdict_zerocopy(self) -> Dict[str, Any]:
92 """Similar to dataclasses.asdict, but avoids deepcopying."""
vllm/model_executor/layers/activation.py
+2 -2
@@ -20,8 +20,8 @@ class SiluAndMul(nn.Module):
20 The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[-1] // 2.
21
22 Shapes:
23 - x: (batch_size, seq_len, 2 * d) or (num_tokens, 2 * d)
24 - return: (batch_size, seq_len, d) or (num_tokens, d)
23 + x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d)
24 + return: (num_tokens, d) or (batch_size, seq_len, d)
25 """
26
27 def _forward(self, x: torch.Tensor) -> torch.Tensor:
vllm/model_executor/layers/attention/attention.py
+2 -1
@@ -17,11 +17,12 @@ class Attention(nn.Module):
17
18 This class takes query, key, and value tensors as input. The input tensors
19 can either contain prompt tokens or generation tokens.
20 +
21 The class does the following:
22
23 1. Store the input key and value tensors in the KV cache.
24 2. Perform (multi-head/multi-query/grouped-query) attention.
24 - 3. Return the output tensor.
25 + 3. Output the output tensor.
26 """
27
28 def __init__(
vllm/model_executor/layers/attention/backends/flash_attn.py
+32 -14
@@ -1,7 +1,7 @@
1 """Attention layer with Flash and PagedAttention."""
2 from typing import List, Optional
3
4 -from flash_attn import flash_attn_func
4 +from flash_attn import flash_attn_varlen_func
5 import torch
6
7 from vllm.model_executor.input_metadata import InputMetadata
@@ -10,6 +10,21 @@ from vllm.model_executor.layers.attention.ops.paged_attn import (
10
11
12 class FlashAttentionBackend:
13 + """
14 + If the input tensors contain prompt tokens, the layout is as follows:
15 + |<--------------- num_prompt_tokens -------------->|
16 + |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->|
17 +
18 + Otherwise, the layout is as follows:
19 + |<------------------ num_generation_tokens (M) ----------------->|
20 + |<--generation_0-->|..........|<--generation_M-1-->|<--padding-->|
21 +
22 + Generation tokens can contain padding when cuda-graph is used.
23 + Currently, prompt tokens don't contain any padding.
24 +
25 + The prompts might have different lengths, while the generation tokens
26 + always have length 1.
27 + """
28
29 def __init__(
30 self,
@@ -52,18 +67,18 @@ class FlashAttentionBackend:
67 """Forward pass with FlashAttention and PagedAttention.
68
69 Args:
55 - query: shape = [batch_size, seq_len, num_heads * head_size]
56 - key: shape = [batch_size, seq_len, num_kv_heads * head_size]
57 - value: shape = [batch_size, seq_len, num_kv_heads * head_size]
70 + query: shape = [num_tokens, num_heads * head_size]
71 + key: shape = [num_tokens, num_kv_heads * head_size]
72 + value: shape = [num_tokens, num_kv_heads * head_size]
73 key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
74 block_size, x]
75 value_cache: shape = [num_blocks, num_kv_heads, head_size,
76 block_size]
77 input_metadata: metadata for the inputs.
78 Returns:
64 - shape = [batch_size, seq_len, num_heads * head_size]
79 + shape = [num_tokens, num_heads * head_size]
80 """
66 - batch_size, seq_len, hidden_size = query.shape
81 + num_tokens, hidden_size = query.shape
82 # Reshape the query, key, and value tensors.
83 query = query.view(-1, self.num_heads, self.head_size)
84 key = key.view(-1, self.num_kv_heads, self.head_size)
@@ -82,13 +97,16 @@ class FlashAttentionBackend:
97 if (key_cache is None or value_cache is None
98 or input_metadata.block_tables.numel() == 0):
99 # normal attention
85 - query = query.unflatten(0, (batch_size, seq_len))
86 - key = key.unflatten(0, (batch_size, seq_len))
87 - value = value.unflatten(0, (batch_size, seq_len))
88 - output = flash_attn_func(
89 - query,
90 - key,
91 - value,
100 + # When block_tables are not filled, it means q and k are the
101 + # prompt, and they have the same length.
102 + output = flash_attn_varlen_func(
103 + q=query,
104 + k=key,
105 + v=value,
106 + cu_seqlens_q=input_metadata.seq_start_loc,
107 + cu_seqlens_k=input_metadata.seq_start_loc,
108 + max_seqlen_q=input_metadata.max_seq_len,
109 + max_seqlen_k=input_metadata.max_seq_len,
110 softmax_scale=self.scale,
111 causal=True,
112 window_size=self.sliding_window,
@@ -118,4 +136,4 @@ class FlashAttentionBackend:
136 )
137
138 # Reshape the output tensor.
121 - return output.view(batch_size, seq_len, hidden_size)
139 + return output.view(num_tokens, hidden_size)
vllm/model_executor/layers/attention/backends/xformers.py
+147 -85
@@ -14,6 +14,21 @@ from vllm.utils import is_hip
14
15
16 class XFormersBackend:
17 + """
18 + If the input tensors contain prompt tokens, the layout is as follows:
19 + |<--------------- num_prompt_tokens --------------->|
20 + |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1--->|
21 +
22 + Otherwise, the layout is as follows:
23 + |<------------------ num_generation_tokens (M) ----------------->|
24 + |<--generation_0-->|..........|<--generation_M-1-->|<--padding-->|
25 +
26 + Generation tokens can contain padding when cuda-graph is used.
27 + Currently, prompt tokens don't contain any padding.
28 +
29 + The prompts might have different lengths, while the generation tokens
30 + always have length 1.
31 + """
32
33 def __init__(
34 self,
@@ -55,19 +70,18 @@ class XFormersBackend:
70 """Forward pass with xFormers and PagedAttention.
71
72 Args:
58 - query: shape = [batch_size, seq_len, num_heads * head_size]
59 - key: shape = [batch_size, seq_len, num_kv_heads * head_size]
60 - value: shape = [batch_size, seq_len, num_kv_heads * head_size]
73 + query: shape = [num_tokens, num_heads * head_size]
74 + key: shape = [num_tokens, num_kv_heads * head_size]
75 + value: shape = [num_tokens, num_kv_heads * head_size]
76 key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
77 block_size, x]
78 value_cache: shape = [num_blocks, num_kv_heads, head_size,
79 block_size]
80 input_metadata: metadata for the inputs.
81 Returns:
67 - shape = [batch_size, seq_len, num_heads * head_size]
82 + shape = [num_tokens, num_heads * head_size]
83 """
69 - batch_size, seq_len, hidden_size = query.shape
70 - # Reshape the query, key, and value tensors.
84 + num_tokens, hidden_size = query.shape
85 query = query.view(-1, self.num_heads, self.head_size)
86 key = key.view(-1, self.num_kv_heads, self.head_size)
87 value = value.view(-1, self.num_kv_heads, self.head_size)
@@ -82,9 +96,10 @@ class XFormersBackend:
96
97 if input_metadata.is_prompt:
98 # Prompt run.
99 + # key_cache and value_cache are None when it is a profiling run.
100 + # block tables are empty if the prompt has never been computed.
101 if (key_cache is None or value_cache is None
102 or input_metadata.block_tables.numel() == 0):
87 - # normal attention
103 if self.num_kv_heads != self.num_heads:
104 # As of Nov 2023, xformers only supports MHA. For MQA/GQA,
105 # project the key and value tensors to the desired number of
@@ -103,61 +118,33 @@ class XFormersBackend:
118 self.num_queries_per_kv,
119 value.shape[-1])
120
106 - # Set attention bias if not provided. This typically happens at
107 - # the very attention layer of every iteration.
108 - # FIXME(woosuk): This is a hack.
109 - if input_metadata.attn_bias is None:
110 - if self.alibi_slopes is None:
111 - attn_bias = BlockDiagonalCausalMask.from_seqlens(
112 - [seq_len] * batch_size)
113 - if self.sliding_window is not None:
114 - attn_bias = attn_bias.make_local_attention(
115 - self.sliding_window)
116 - input_metadata.attn_bias = attn_bias
117 - else:
118 - input_metadata.attn_bias = _make_alibi_bias(
119 - self.alibi_slopes, self.num_kv_heads, batch_size,
120 - seq_len, query.dtype)
121 -
121 if self.use_ref_attention:
123 - output = _ref_masked_attention(
124 - query,
125 - key,
126 - value,
127 - self.num_heads,
128 - self.num_kv_heads,
129 - self.head_size,
130 - self.scale,
131 - )
122 + print("ref attention used.")
123 + output = torch.empty_like(query)
124 + start = 0
125 + for _, prompt_len in enumerate(input_metadata.prompt_lens):
126 + end = start + prompt_len
127 + out = _ref_masked_attention(
128 + query[None, start:end],
129 + key[None, start:end],
130 + value[None, start:end],
131 + self.num_heads,
132 + self.num_kv_heads,
133 + self.head_size,
134 + self.scale,
135 + )
136 + # TODO(woosuk): Unnecessary copy. Optimize.
137 + output[start:end].copy_(out)
138 + start += prompt_len
139 +
140 # Using view got RuntimeError: view size is not compatible
141 # with input tensor's size and stride (at least one
142 # dimension spans across two contiguous subspaces).
143 # Use reshape instead.
136 - return output.reshape(batch_size, seq_len, hidden_size)
137 -
138 - # TODO(woosuk): Too many view operations. Let's try to reduce
139 - # them in the future for code readability.
140 - if self.alibi_slopes is None:
141 - query = query.unsqueeze(0)
142 - key = key.unsqueeze(0)
143 - value = value.unsqueeze(0)
144 - else:
145 - query = query.unflatten(0, (batch_size, seq_len))
146 - key = key.unflatten(0, (batch_size, seq_len))
147 - value = value.unflatten(0, (batch_size, seq_len))
148 -
149 - out = xops.memory_efficient_attention_forward(
150 - query,
151 - key,
152 - value,
153 - attn_bias=input_metadata.attn_bias,
154 - p=0.0,
155 - scale=self.scale,
156 - op=xops.fmha.MemoryEfficientAttentionFlashAttentionOp[0] if
157 - (is_hip()) else None,
158 - )
159 - output = out.view_as(query)
144 + return output.reshape(num_tokens, hidden_size)
145
146 + output = self._run_memory_efficient_xformer_forward(
147 + query, key, value, input_metadata)
148 else:
149 # prefix-enabled attention
150 output = PagedAttentionImpl.forward_prefix(
@@ -182,41 +169,117 @@ class XFormersBackend:
169 )
170
171 # Reshape the output tensor.
185 - return output.view(batch_size, seq_len, hidden_size)
172 + return output.view(-1, self.num_heads * self.head_size)
173 +
174 + def _run_memory_efficient_xformer_forward(
175 + self,
176 + query: torch.Tensor,
177 + key: torch.Tensor,
178 + value: torch.Tensor,
179 + input_metadata: InputMetadata,
180 + ) -> torch.Tensor:
181 + """Attention for 1D query of multiple prompts. Multiple prompt
182 + tokens are flattened in to `query` input.
183 +
184 + Args:
185 + output: shape = [num_prompt_tokens, num_heads, head_size]
186 + query: shape = [num_prompt_tokens, num_heads, head_size]
187 + key: shape = [num_prompt_tokens, num_kv_heads, head_size]
188 + value: shape = [num_prompt_tokens, num_kv_heads, head_size]
189 + input_metadata: metadata for paged attention.
190 + """
191 + # Set attention bias if not provided. This typically happens at
192 + # the very attention layer of every iteration.
193 + # FIXME(woosuk): This is a hack.
194 + if input_metadata.attn_bias is None:
195 + if self.alibi_slopes is None:
196 + attn_bias = BlockDiagonalCausalMask.from_seqlens(
197 + input_metadata.prompt_lens)
198 + if self.sliding_window is not None:
199 + attn_bias = attn_bias.make_local_attention(
200 + self.sliding_window)
201 + input_metadata.attn_bias = [attn_bias]
202 + else:
203 + input_metadata.attn_bias = _make_alibi_bias(
204 + self.alibi_slopes, self.num_kv_heads, query.dtype,
205 + input_metadata)
206 +
207 + op = xops.fmha.MemoryEfficientAttentionFlashAttentionOp[0] if (
208 + is_hip()) else None
209 + # No alibi slopes.
210 + # TODO(woosuk): Too many view operations. Let's try to reduce
211 + # them in the future for code readability.
212 + if self.alibi_slopes is None:
213 + query = query.unsqueeze(0)
214 + key = key.unsqueeze(0)
215 + value = value.unsqueeze(0)
216 + out = xops.memory_efficient_attention_forward(
217 + query,
218 + key,
219 + value,
220 + attn_bias=input_metadata.attn_bias[0],
221 + p=0.0,
222 + scale=self.scale,
223 + op=op)
224 +
225 + return out.view_as(query)
226 +
227 + # Attention with alibi slopes.
228 + # FIXME(woosuk): Because xformers does not support dynamic sequence
229 + # lengths with custom attention bias, we process each prompt one by
230 + # one. This is inefficient, especially when we have many short prompts.
231 + output = torch.empty_like(query)
232 + start = 0
233 + for i, prompt_len in enumerate(input_metadata.prompt_lens):
234 + end = start + prompt_len
235 + out = xops.memory_efficient_attention_forward(
236 + query[None, start:end],
237 + key[None, start:end],
238 + value[None, start:end],
239 + attn_bias=input_metadata.attn_bias[i],
240 + p=0.0,
241 + scale=self.scale,
242 + op=op)
243 + # TODO(woosuk): Unnecessary copy. Optimize.
244 + output[start:end].copy_(out.squeeze(0))
245 + start += prompt_len
246 + return output
247
248
249 def _make_alibi_bias(
250 alibi_slopes: torch.Tensor,
251 num_kv_heads: int,
191 - batch_size: int,
192 - seq_len: int,
252 dtype: torch.dtype,
253 + input_metadata: InputMetadata,
254 ) -> LowerTriangularMaskWithTensorBias:
195 - bias = torch.arange(seq_len, dtype=dtype)
196 - # NOTE(zhuohan): HF uses
197 - # `bias = bias[None, :].repeat(prompt_len, 1)`
198 - # here. We find that both biases give the same results, but
199 - # the bias below more accurately follows the original ALiBi
200 - # paper.
201 - bias = bias[None, :] - bias[:, None]
202 -
203 - # When using custom attention bias, xformers requires the bias to
204 - # be sliced from a tensor whose length is a multiple of 8.
205 - padded_len = (seq_len + 7) // 8 * 8
206 - num_heads = alibi_slopes.shape[0]
207 - bias = torch.empty(
208 - batch_size,
209 - num_heads,
210 - seq_len,
211 - padded_len,
212 - device=alibi_slopes.device,
213 - dtype=dtype,
214 - )[:, :, :, :seq_len].copy_(bias)
215 - bias.mul_(alibi_slopes[:, None, None])
216 - if num_heads != num_kv_heads:
217 - bias = bias.unflatten(1, (num_kv_heads, num_heads // num_kv_heads))
218 - attn_bias = LowerTriangularMaskWithTensorBias(bias)
219 - return attn_bias
255 + attn_biases = []
256 + for prompt_len in input_metadata.prompt_lens:
257 + bias = torch.arange(prompt_len, dtype=dtype)
258 + # NOTE(zhuohan): HF uses
259 + # `bias = bias[None, :].repeat(prompt_len, 1)`
260 + # here. We find that both biases give the same results, but
261 + # the bias below more accurately follows the original ALiBi
262 + # paper.
263 + # Calculate a matrix where each element represents ith element- jth
264 + # element.
265 + bias = bias[None, :] - bias[:, None]
266 +
267 + padded_len = (prompt_len + 7) // 8 * 8
268 + num_heads = alibi_slopes.shape[0]
269 + bias = torch.empty(
270 + 1, # batch size
271 + num_heads,
272 + prompt_len,
273 + padded_len,
274 + device=alibi_slopes.device,
275 + dtype=dtype,
276 + )[:, :, :, :prompt_len].copy_(bias)
277 + bias.mul_(alibi_slopes[:, None, None])
278 + if num_heads != num_kv_heads:
279 + bias = bias.unflatten(1, (num_kv_heads, num_heads // num_kv_heads))
280 + attn_biases.append(LowerTriangularMaskWithTensorBias(bias))
281 +
282 + return attn_biases
283
284
285 def _check_use_ref_attention() -> bool:
@@ -239,7 +302,6 @@ def _ref_masked_attention(
302 query = query.view(-1, num_heads, head_size)
303 key = key.view(-1, num_kv_heads, head_size)
304 value = value.view(-1, num_kv_heads, head_size)
242 -
305 seq_len, _, _ = query.shape
306 attn_mask = torch.triu(torch.ones(seq_len,
307 seq_len,
vllm/model_executor/layers/attention/ops/paged_attn.py
+5 -4
@@ -128,11 +128,12 @@ class PagedAttentionImpl:
128 output,
129 key_cache,
130 value_cache,
131 - input_metadata.block_tables, # [BS, max_block_per_request]
132 - input_metadata.start_loc,
133 - input_metadata.prompt_lens,
131 + input_metadata.block_tables,
132 + # subquery_start_loc is (batch_size + 1,)
133 + input_metadata.subquery_start_loc[:-1],
134 + input_metadata.prompt_lens_tensor,
135 input_metadata.context_lens,
135 - input_metadata.max_seq_len,
136 + input_metadata.max_subquery_len,
137 alibi_slopes,
138 )
139 return output
vllm/model_executor/layers/sampler.py
-1
@@ -128,7 +128,6 @@ def _prune_hidden_states(
128 hidden_states: torch.Tensor,
129 sampling_metadata: SamplingMetadata,
130 ) -> torch.Tensor:
131 - hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
131 return hidden_states.index_select(0,
132 sampling_metadata.selected_token_indices)
133
vllm/worker/model_runner.py
+144 -95
@@ -28,9 +28,12 @@ logger = init_logger(__name__)
28 KVCache = Tuple[torch.Tensor, torch.Tensor]
29 _PAD_SLOT_ID = -1
30 LORA_WARMUP_RANK = 8
31 -# Capture graphs for batch size 1, 2, 4, 8, 16, 24, 32, 40, ..., 256.
31 +_BATCH_SIZE_ALIGNMENT = 8
32 +# Capture graphs for token size 1, 2, 4, 8, 16, 24, 32, 40, ..., 256.
33 # NOTE: _get_graph_batch_size needs to be updated if this list is changed.
33 -_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [8 * i for i in range(1, 33)]
34 +_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
35 + _BATCH_SIZE_ALIGNMENT * i for i in range(1, 33)
36 +]
37
38
39 class ModelRunner:
@@ -107,8 +110,7 @@ class ModelRunner:
110 ), "Model does not have embedding_padding_modules"
111 self.lora_manager = LRUCacheWorkerLoRAManager(
112 self.scheduler_config.max_num_seqs,
110 - self.scheduler_config.max_num_batched_tokens +
111 - self.scheduler_config.max_paddings, self.vocab_size,
113 + self.scheduler_config.max_num_batched_tokens, self.vocab_size,
114 self.lora_config, self.device, self.model.embedding_modules,
115 self.model.embedding_padding_modules)
116 self.model = self.lora_manager.create_lora_manager(self.model)
@@ -116,10 +118,13 @@ class ModelRunner:
118 def set_block_size(self, block_size: int) -> None:
119 self.block_size = block_size
120
119 - max_num_blocks = (self.max_context_len_to_capture + block_size -
120 - 1) // block_size
121 self.graph_block_tables = np.zeros(
122 - (max(_BATCH_SIZES_TO_CAPTURE), max_num_blocks), dtype=np.int32)
122 + (max(_BATCH_SIZES_TO_CAPTURE), self.get_max_block_per_batch()),
123 + dtype=np.int32)
124 +
125 + def get_max_block_per_batch(self) -> int:
126 + block_size = self.block_size
127 + return (self.max_context_len_to_capture + block_size - 1) // block_size
128
129 def _prepare_prompt(
130 self,
@@ -127,9 +132,9 @@ class ModelRunner:
132 ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, List[int], List[int],
133 List[int], List[int], Set[LoRARequest]]:
134 assert len(seq_group_metadata_list) > 0
130 - input_tokens: List[List[int]] = []
131 - input_positions: List[List[int]] = []
132 - slot_mapping: List[List[int]] = []
135 + input_tokens: List[int] = []
136 + input_positions: List[int] = []
137 + slot_mapping: List[int] = []
138 lora_index_mapping: List[int] = []
139 lora_prompt_mapping: List[int] = []
140 lora_requests: Set[LoRARequest] = set()
@@ -158,16 +163,18 @@ class ModelRunner:
163 computed_len = len(computed_block_nums) * self.block_size
164 prompt_tokens = prompt_tokens[computed_len:]
165 prefix_block_tables.append(computed_block_nums)
166 + context_len = computed_len
167 else:
168 prefix_block_tables.append([])
169 + context_len = 0
170 # actual prompt lens
164 - context_lens.append(computed_len)
171 + context_lens.append(context_len)
172 subquery_lens.append(prompt_len - computed_len)
173
167 - input_tokens.append(prompt_tokens)
174 + input_tokens.extend(prompt_tokens)
175 # NOTE(woosuk): Here we assume that the first token in the prompt
176 # is always the first token in the sequence.
170 - input_positions.append(
177 + input_positions.extend(
178 list(range(computed_len, computed_len + len(prompt_tokens))))
179
180 lora_id = seq_group_metadata.lora_int_id
@@ -175,7 +182,7 @@ class ModelRunner:
182 if lora_id > 0:
183 lora_requests.add(seq_group_metadata.lora_request)
184
178 - lora_index_mapping.append([lora_id] * (prompt_len - computed_len))
185 + lora_index_mapping += [lora_id] * (prompt_len - computed_len)
186 lora_prompt_mapping.extend(
187 [lora_id] *
188 (prompt_len - computed_len
@@ -184,11 +191,10 @@ class ModelRunner:
191 if seq_group_metadata.block_tables is None:
192 # During memory profiling, the block tables are not initialized
193 # yet. In this case, we just use a dummy slot mapping.
187 - slot_mapping.append([_PAD_SLOT_ID] * prompt_len)
194 + slot_mapping.extend([_PAD_SLOT_ID] * prompt_len)
195 continue
196
197 # Compute the slot mapping.
191 - slot_mapping.append([])
198 block_table = seq_group_metadata.block_tables[seq_id]
199 # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
200 # where start_idx is max(0, prompt_len - sliding_window).
@@ -203,35 +209,30 @@ class ModelRunner:
209 start_idx = max(0, prompt_len - self.sliding_window)
210 for i in range(computed_len, prompt_len):
211 if i < start_idx:
206 - slot_mapping[-1].append(_PAD_SLOT_ID)
212 + slot_mapping.append(_PAD_SLOT_ID)
213 continue
214
215 block_number = block_table[i // self.block_size]
216 block_offset = i % self.block_size
217 slot = block_number * self.block_size + block_offset
212 - slot_mapping[-1].append(slot)
213 -
214 - max_prompt_len = max(subquery_lens)
215 - assert max_prompt_len > 0
216 - input_tokens = _make_tensor_with_pad(input_tokens,
217 - max_prompt_len,
218 - pad=0,
219 - dtype=torch.long,
220 - device=self.device)
221 - input_positions = _make_tensor_with_pad(input_positions,
222 - max_prompt_len,
223 - pad=0,
224 - dtype=torch.long,
225 - device=self.device)
226 - slot_mapping = _make_tensor_with_pad(slot_mapping,
227 - max_prompt_len,
228 - pad=_PAD_SLOT_ID,
229 - dtype=torch.long,
230 - device=self.device)
231 - lora_index_mapping = [
232 - _pad_to_max(mapping, max_prompt_len, pad=0)
233 - for mapping in lora_index_mapping
234 - ]
218 + slot_mapping.append(slot)
219 +
220 + max_subquery_len = max(subquery_lens)
221 + max_seq_len = max(prompt_lens)
222 + num_prompt_tokens = len(input_tokens)
223 + assert max_subquery_len > 0
224 +
225 + input_tokens = torch.tensor(input_tokens,
226 + dtype=torch.long,
227 + device=self.device)
228 + input_positions = torch.tensor(input_positions,
229 + dtype=torch.long,
230 + device=self.device)
231 + slot_mapping = torch.tensor(slot_mapping,
232 + dtype=torch.long,
233 + device=self.device)
234 + lora_index_mapping = lora_index_mapping
235 +
236 context_lens_tensor = torch.tensor(context_lens,
237 dtype=torch.int,
238 device=self.device)
@@ -244,22 +245,45 @@ class ModelRunner:
245 dtype=torch.int,
246 device=self.device,
247 )
247 - start_loc_tensor = torch.arange(0,
248 - len(prompt_lens) * max_prompt_len,
249 - max_prompt_len,
250 - dtype=torch.long,
251 - device=self.device)
248 +
249 + # Query length can be shorter than key (i.e., prompt) when prefill
250 + # is chunked or prefix cached.
251 + subquery_lens_tensor = torch.tensor(subquery_lens,
252 + dtype=torch.long,
253 + device=self.device)
254 + subquery_start_loc = torch.zeros(subquery_lens_tensor.shape[0] + 1,
255 + dtype=torch.int32,
256 + device=self.device)
257 +
258 prompt_lens_tensor = torch.tensor(prompt_lens,
259 dtype=torch.long,
260 device=self.device)
261 + seq_start_loc = torch.zeros(prompt_lens_tensor.shape[0] + 1,
262 + dtype=torch.int32,
263 + device=self.device)
264 +
265 + torch.cumsum(subquery_lens_tensor,
266 + dim=0,
267 + dtype=subquery_start_loc.dtype,
268 + out=subquery_start_loc[1:])
269 +
270 + torch.cumsum(prompt_lens_tensor,
271 + dim=0,
272 + dtype=seq_start_loc.dtype,
273 + out=seq_start_loc[1:])
274
275 input_metadata = InputMetadata(
276 is_prompt=True,
277 slot_mapping=slot_mapping,
259 - prompt_lens=prompt_lens_tensor,
260 - max_seq_len=max_prompt_len,
261 - start_loc=start_loc_tensor,
278 + prompt_lens=prompt_lens,
279 + prompt_lens_tensor=prompt_lens_tensor,
280 + num_prompt_tokens=num_prompt_tokens,
281 + num_generation_tokens=0,
282 + max_subquery_len=max_subquery_len,
283 max_context_len=None,
284 + max_seq_len=max_seq_len,
285 + subquery_start_loc=subquery_start_loc,
286 + seq_start_loc=seq_start_loc,
287 context_lens=context_lens_tensor,
288 block_tables=block_tables,
289 use_cuda_graph=False,
@@ -275,9 +299,9 @@ class ModelRunner:
299 ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, List[int], List[int],
300 Set[LoRARequest]]:
301 assert len(seq_group_metadata_list) > 0
278 - input_tokens: List[List[int]] = []
279 - input_positions: List[List[int]] = []
280 - slot_mapping: List[List[int]] = []
302 + input_tokens: List[int] = []
303 + input_positions: List[int] = []
304 + slot_mapping: List[int] = []
305 context_lens: List[int] = []
306 block_tables: List[List[int]] = []
307 lora_index_mapping: List[int] = []
@@ -296,11 +320,11 @@ class ModelRunner:
320 for seq_id in seq_ids:
321 seq_data = seq_group_metadata.seq_data[seq_id]
322 generation_token = seq_data.get_last_token_id()
299 - input_tokens.append([generation_token])
323 + input_tokens.append(generation_token)
324
325 seq_len = seq_data.get_len()
326 position = seq_len - 1
303 - input_positions.append([position])
327 + input_positions.append(position)
328
329 context_len = seq_len if self.sliding_window is None else min(
330 seq_len, self.sliding_window)
@@ -310,8 +334,8 @@ class ModelRunner:
334 block_number = block_table[position // self.block_size]
335 block_offset = position % self.block_size
336 slot = block_number * self.block_size + block_offset
313 - slot_mapping.append([slot])
314 - lora_index_mapping.append([lora_id])
337 + slot_mapping.append(slot)
338 + lora_index_mapping.append(lora_id)
339 lora_prompt_mapping.append(lora_id)
340
341 if self.sliding_window is not None:
@@ -320,6 +344,9 @@ class ModelRunner:
344 block_table = block_table[-sliding_window_blocks:]
345 block_tables.append(block_table)
346
347 + # vLLM uses cuda graph only for decoding requests.
348 + # See `capture_model` API for more details.
349 + # For decoding requests, batch_size == input_tokens.
350 batch_size = len(input_tokens)
351 max_context_len = max(context_lens)
352 use_captured_graph = (
@@ -327,38 +354,37 @@ class ModelRunner:
354 and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
355 and max_context_len <= self.max_context_len_to_capture)
356 if use_captured_graph:
330 - # Pad the input tokens, positions, and slot mapping to match the
331 - # batch size of the captured graph.
357 graph_batch_size = _get_graph_batch_size(batch_size)
358 assert graph_batch_size >= batch_size
359 for _ in range(graph_batch_size - batch_size):
335 - input_tokens.append([])
336 - input_positions.append([])
337 - slot_mapping.append([])
360 + input_tokens.append(0)
361 + input_positions.append(0)
362 + slot_mapping.append(_PAD_SLOT_ID)
363 context_lens.append(1)
364 block_tables.append([])
365 + lora_index_mapping.append(0)
366 batch_size = graph_batch_size
367
342 - input_tokens = _make_tensor_with_pad(input_tokens,
343 - max_len=1,
344 - pad=0,
345 - dtype=torch.long,
346 - device=self.device)
347 - input_positions = _make_tensor_with_pad(input_positions,
348 - max_len=1,
349 - pad=0,
350 - dtype=torch.long,
351 - device=self.device)
352 - slot_mapping = _make_tensor_with_pad(slot_mapping,
353 - max_len=1,
354 - pad=_PAD_SLOT_ID,
355 - dtype=torch.long,
356 - device=self.device)
368 + input_tokens = torch.tensor(input_tokens,
369 + dtype=torch.long,
370 + device=self.device)
371 + input_positions = torch.tensor(input_positions,
372 + dtype=torch.long,
373 + device=self.device)
374 + slot_mapping = torch.tensor(slot_mapping,
375 + dtype=torch.long,
376 + device=self.device)
377 context_lens = torch.tensor(context_lens,
378 dtype=torch.int,
379 device=self.device)
380
381 if use_captured_graph:
382 + # When using cuda-graph all these tensors should be
383 + # padded.
384 + assert context_lens.shape[0] == input_tokens.shape[0]
385 + assert context_lens.shape[0] == input_positions.shape[0]
386 + assert context_lens.shape[0] == slot_mapping.shape[0]
387 +
388 # The shape of graph_block_tables is
389 # [max batch size, max context len // block size].
390 input_block_tables = self.graph_block_tables[:batch_size]
@@ -377,17 +403,18 @@ class ModelRunner:
403 device=self.device,
404 )
405
380 - lora_index_mapping = [
381 - _pad_to_max(mapping, 1, pad=0) for mapping in lora_index_mapping
382 - ]
383 -
406 input_metadata = InputMetadata(
407 is_prompt=False,
408 slot_mapping=slot_mapping,
409 prompt_lens=None,
388 - max_seq_len=None,
389 - start_loc=None,
410 + prompt_lens_tensor=None,
411 + num_prompt_tokens=0,
412 + num_generation_tokens=len(input_tokens),
413 + max_subquery_len=None,
414 max_context_len=max_context_len,
415 + max_seq_len=None,
416 + subquery_start_loc=None,
417 + seq_start_loc=None,
418 context_lens=context_lens,
419 block_tables=block_tables,
420 use_cuda_graph=use_captured_graph,
@@ -411,7 +438,6 @@ class ModelRunner:
438 categorized_sampled_token_indices_start_idx = 0
439 pin_memory = not self.in_wsl and not self.device_config.is_neuron
440
414 - max_subquery_len = max(subquery_lens) if subquery_lens else 1
441 for i, seq_group_metadata in enumerate(seq_group_metadata_list):
442 seq_ids = list(seq_group_metadata.seq_data.keys())
443 sampling_params = seq_group_metadata.sampling_params
@@ -439,7 +465,7 @@ class ModelRunner:
465 selected_token_start_idx + subquery_len - 1))
466 selected_token_indices.append(selected_token_start_idx +
467 subquery_len - 1)
442 - selected_token_start_idx += max_subquery_len
468 + selected_token_start_idx += subquery_len
469
470 if sampling_params.seed is not None:
471 seq_group_metadata.state.generator = torch.Generator(
@@ -521,11 +547,8 @@ class ModelRunner:
547 subquery_lens)
548
549 if self.lora_config:
524 - flat_lora_index_mapping = [
525 - item for sublist in lora_index_mapping for item in sublist
526 - ]
550 lora_mapping = LoRAMapping(
528 - flat_lora_index_mapping,
551 + lora_index_mapping,
552 lora_prompt_mapping,
553 )
554 else:
@@ -679,6 +702,18 @@ class ModelRunner:
702
703 @torch.inference_mode()
704 def capture_model(self, kv_caches: List[KVCache]) -> None:
705 + """Cuda graph capture a model.
706 +
707 + Note that CUDA graph's performance gain is negligible if number
708 + of batched tokens are larger than 200. And since CUDA graph
709 + requires fixed sized tensors, supporting large/variable batch
710 + size requires high GPU memory overhead. Thus, vLLM only captures
711 + decoding requests. Mixed batch (chunked prefill + decoding) or
712 + prefill requests are not captured.
713 +
714 + Since it is used for decoding-only, it assumes there's only 1 token
715 + per sequence in the batch.
716 + """
717 # NOTE(woosuk): This is a hack to ensure that the NCCL backend is never
718 # deleted before the CUDA graphs.
719 self.cupy_nccl_backend = cupy_utils.get_nccl_backend()
@@ -697,10 +732,9 @@ class ModelRunner:
732
733 # Prepare dummy inputs. These will be reused for all batch sizes.
734 max_batch_size = max(_BATCH_SIZES_TO_CAPTURE)
700 - input_tokens = torch.zeros(max_batch_size, 1, dtype=torch.long).cuda()
701 - input_positions = torch.zeros(max_batch_size, 1,
702 - dtype=torch.long).cuda()
703 - slot_mapping = torch.empty(max_batch_size, 1, dtype=torch.long).cuda()
735 + input_tokens = torch.zeros(max_batch_size, dtype=torch.long).cuda()
736 + input_positions = torch.zeros(max_batch_size, dtype=torch.long).cuda()
737 + slot_mapping = torch.empty(max_batch_size, dtype=torch.long).cuda()
738 slot_mapping.fill_(_PAD_SLOT_ID)
739 context_lens = torch.ones(max_batch_size, dtype=torch.int32).cuda()
740 block_tables = torch.from_numpy(self.graph_block_tables).cuda()
@@ -726,9 +760,14 @@ class ModelRunner:
760 is_prompt=False,
761 slot_mapping=slot_mapping[:batch_size],
762 prompt_lens=None,
729 - max_seq_len=None,
730 - start_loc=None,
763 + prompt_lens_tensor=None,
764 + num_prompt_tokens=0,
765 + num_generation_tokens=batch_size,
766 + max_subquery_len=None,
767 max_context_len=self.max_context_len_to_capture,
768 + max_seq_len=None,
769 + subquery_start_loc=None,
770 + seq_start_loc=None,
771 context_lens=context_lens[:batch_size],
772 block_tables=block_tables[:batch_size],
773 use_cuda_graph=True,
@@ -845,7 +884,6 @@ class CUDAGraphRunner:
884 non_blocking=True)
885 self.input_buffers["block_tables"].copy_(input_metadata.block_tables,
886 non_blocking=True)
848 -
887 # Run the graph.
888 self.graph.replay()
889
@@ -877,17 +915,28 @@ def _make_tensor_with_pad(
915 dtype: torch.dtype,
916 device: Optional[Union[str, torch.device]],
917 ) -> torch.Tensor:
918 + """Make a padded tensor of a 2D inputs.
919 +
920 + The padding is applied to the end of each inner list until it reaches
921 + `max_len`.
922 + """
923 padded_x = [_pad_to_max(x_i, max_len, pad) for x_i in x]
924 return torch.tensor(padded_x, dtype=dtype, device=device)
925
926
927 def _get_graph_batch_size(batch_size: int) -> int:
928 + """Returns the padded batch size given actual batch size.
929 +
930 + Batch sizes are 1, 2, 4, _BATCH_SIZE_ALIGNMENT,
931 + 2*_BATCH_SIZE_ALIGNMENT, 3*_BATCH_SIZE_ALIGNMENT...
932 + """
933 if batch_size <= 2:
934 return batch_size
935 elif batch_size <= 4:
936 return 4
937 else:
890 - return (batch_size + 7) // 8 * 8
938 + return ((batch_size + _BATCH_SIZE_ALIGNMENT - 1) //
939 + _BATCH_SIZE_ALIGNMENT * _BATCH_SIZE_ALIGNMENT)
940
941
942 def _async_h2d(