A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[Experimental] Prefix Caching Support (#1669)
Co-authored-by: DouHappy <2278958187@qq.com> Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
shiyi.c_98 committed
Jan 17, 2024 at 16:32 UTC
d10f8e1d43bfb0656b6848ad0c681ecbdec812d6
20 files changed
+1356
-71
.buildkite/test-pipeline.yaml
+4
@@ -31,6 +31,10 @@ steps:
31
- pytest -v -s models --forked
32
soft_fail: true
33
34
+- label: Prefix Caching Test
35
+ commands:
36
+ - pytest -v -s prefix_caching
37
+
38
- label: Samplers Test
39
command: pytest -v -s samplers --forked
40
examples/offline_inference_with_prefix.py
new
+51
@@ -0,0 +1,51 @@
1
+from vllm import LLM, SamplingParams
2
+
3
+prefix = (
4
+ "You are an expert school principal, skilled in effectively managing "
5
+ "faculty and staff. Draft 10-15 questions for a potential first grade "
6
+ "Head Teacher for my K-12, all-girls', independent school that emphasizes "
7
+ "community, joyful discovery, and life-long learning. The candidate is "
8
+ "coming in for a first-round panel interview for a 8th grade Math "
9
+ "teaching role. They have 5 years of previous teaching experience "
10
+ "as an assistant teacher at a co-ed, public school with experience "
11
+ "in middle school math teaching. Based on these information, fulfill "
12
+ "the following paragraph: ")
13
+
14
+# Sample prompts.
15
+prompts = [
16
+ "Hello, my name is",
17
+ "The president of the United States is",
18
+ "The capital of France is",
19
+ "The future of AI is",
20
+]
21
+# Create a sampling params object.
22
+sampling_params = SamplingParams(temperature=0.0)
23
+
24
+# Create an LLM.
25
+llm = LLM(model="facebook/opt-125m")
26
+
27
+generating_prompts = [prefix + prompt for prompt in prompts]
28
+
29
+# Generate texts from the prompts. The output is a list of RequestOutput objects
30
+# that contain the prompt, generated text, and other information.
31
+outputs = llm.generate(generating_prompts, sampling_params)
32
+# Print the outputs.
33
+for output in outputs:
34
+ prompt = output.prompt
35
+ generated_text = output.outputs[0].text
36
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
37
+
38
+print("-" * 80)
39
+
40
+# -1 since the last token can change when concatenating prompts.
41
+prefix_pos = len(llm.llm_engine.tokenizer.encode(prefix)) - 1
42
+
43
+# Generate with prefix
44
+outputs = llm.generate(generating_prompts, sampling_params,
45
+ prefix_pos=[prefix_pos] * len(generating_prompts))
46
+
47
+# Print the outputs. You should see the same outputs as before
48
+for output in outputs:
49
+ prompt = output.prompt
50
+ generated_text = output.outputs[0].text
51
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
tests/kernels/test_prefix_prefill.py
new
+168
@@ -0,0 +1,168 @@
1
+import random
2
+import pytest
3
+import time
4
+
5
+import torch
6
+from vllm.model_executor.layers.triton_kernel.prefix_prefill import (
7
+ context_attention_fwd)
8
+from xformers import ops as xops
9
+from xformers.ops.fmha.attn_bias import BlockDiagonalCausalFromBottomRightMask
10
+
11
+NUM_HEADS = [12]
12
+HEAD_SIZES = [128]
13
+DTYPES = [torch.float16]
14
+
15
+
16
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
17
+@pytest.mark.parametrize("head_size", HEAD_SIZES)
18
+@pytest.mark.parametrize("dtype", DTYPES)
19
+@torch.inference_mode()
20
+def test_contexted_kv_attention(
21
+ num_heads: int,
22
+ head_size: int,
23
+ dtype: torch.dtype,
24
+) -> None:
25
+ random.seed(0)
26
+ torch.manual_seed(0)
27
+ MAX_SEQ_LEN = 1024
28
+ MAX_CTX_LEN = 1024
29
+ BS = 10
30
+ cache_size = 640
31
+ block_size = 32
32
+ max_block_per_request = 64
33
+ subquery_lens = [random.randint(16, MAX_SEQ_LEN) for _ in range(BS)]
34
+ ctx_lens = [random.randint(16, MAX_CTX_LEN) for _ in range(BS)]
35
+ seq_lens = [a + b for a, b in zip(subquery_lens, ctx_lens)]
36
+
37
+ num_tokens = sum(subquery_lens)
38
+ query = torch.empty(num_tokens,
39
+ num_heads,
40
+ head_size,
41
+ dtype=dtype,
42
+ device='cuda')
43
+ query.uniform_(-1e-3, 1e-3)
44
+ output = torch.empty(num_tokens,
45
+ num_heads,
46
+ head_size,
47
+ dtype=dtype,
48
+ device='cuda')
49
+
50
+ kv = torch.empty(sum(seq_lens),
51
+ 2,
52
+ num_heads,
53
+ head_size,
54
+ dtype=dtype,
55
+ device='cuda')
56
+ kv.uniform_(-1e-3, 1e-3)
57
+ key, value = kv.unbind(dim=1)
58
+
59
+ k_cache = torch.zeros(cache_size,
60
+ block_size,
61
+ num_heads,
62
+ head_size,
63
+ dtype=dtype,
64
+ device='cuda')
65
+ v_cache = torch.zeros(cache_size,
66
+ block_size,
67
+ num_heads,
68
+ head_size,
69
+ dtype=dtype,
70
+ device='cuda')
71
+ k = torch.zeros(sum(subquery_lens),
72
+ num_heads,
73
+ head_size,
74
+ dtype=dtype,
75
+ device='cuda')
76
+ v = torch.zeros(sum(subquery_lens),
77
+ num_heads,
78
+ head_size,
79
+ dtype=dtype,
80
+ device='cuda')
81
+ values = torch.arange(0, cache_size, dtype=torch.long, device='cuda')
82
+ values = values[torch.randperm(cache_size)]
83
+ block_table = values[:BS * max_block_per_request].view(
84
+ BS, max_block_per_request)
85
+ b_seq_len = torch.tensor(seq_lens, dtype=torch.long, device='cuda')
86
+ b_ctx_len = torch.tensor(ctx_lens, dtype=torch.long, device='cuda')
87
+ b_start_loc = torch.cumsum(torch.tensor([0] + subquery_lens[:-1],
88
+ dtype=torch.long,
89
+ device='cuda'),
90
+ dim=0)
91
+ max_input_len = MAX_SEQ_LEN
92
+ # copy kv to cache
93
+ b_seq_start_loc = torch.cumsum(torch.tensor([0] + seq_lens[:-1],
94
+ dtype=torch.long,
95
+ device='cuda'),
96
+ dim=0)
97
+ for i in range(BS):
98
+ for j in range(subquery_lens[i]):
99
+ k[b_start_loc[i] + j].copy_(key[b_seq_start_loc[i] + b_ctx_len[i] +
100
+ j])
101
+ v[b_start_loc[i] + j].copy_(value[b_seq_start_loc[i] +
102
+ b_ctx_len[i] + j])
103
+ cur_ctx = 0
104
+ block_id = 0
105
+ while cur_ctx < b_ctx_len[i]:
106
+ start_loc = b_seq_start_loc[i] + cur_ctx
107
+ if cur_ctx + block_size > b_ctx_len[i]:
108
+ end_loc = b_seq_start_loc[i] + b_ctx_len[i]
109
+ else:
110
+ end_loc = start_loc + block_size
111
+ start_slot = block_table[i, block_id] * block_size
112
+ end_slot = start_slot + end_loc - start_loc
113
+ k_cache.view(-1, num_heads, head_size)[start_slot:end_slot].copy_(
114
+ key[start_loc:end_loc])
115
+ v_cache.view(-1, num_heads, head_size)[start_slot:end_slot].copy_(
116
+ value[start_loc:end_loc])
117
+ cur_ctx += block_size
118
+ block_id += 1
119
+ # transpose K_cache[num_blocks, block_size, num_kv_heads, head_size]
120
+ # to K_cache[num_blocks, num_kv_heads, head_size/8, block_size, 8]
121
+ k_cache = k_cache.view(-1, block_size, num_heads, head_size // 8,
122
+ 8).permute(0, 2, 3, 1, 4).contiguous()
123
+ # transpose V_cache[num_blocks, block_size, num_kv_heads, head_size]
124
+ # to V_cache[num_blocks, num_kv_heads, head_size, block_size]
125
+ v_cache = v_cache.view(-1, block_size, num_heads,
126
+ head_size).permute(0, 2, 3, 1).contiguous()
127
+
128
+ context_attention_fwd(query, k, v, output, k_cache, v_cache, block_table,
129
+ b_start_loc, b_seq_len, b_ctx_len, max_input_len)
130
+ torch.cuda.synchronize()
131
+ start_time = time.time()
132
+ context_attention_fwd(query, k, v, output, k_cache, v_cache, block_table,
133
+ b_start_loc, b_seq_len, b_ctx_len, max_input_len)
134
+ torch.cuda.synchronize()
135
+ end_time = time.time()
136
+ print(f"triton Time: {(end_time - start_time)*1000:.2f} ms")
137
+
138
+ scale = float(1.0 / (head_size**0.5))
139
+
140
+ attn_op = xops.fmha.cutlass.FwOp()
141
+
142
+ attn_bias = BlockDiagonalCausalFromBottomRightMask.from_seqlens(
143
+ subquery_lens, seq_lens)
144
+ output_ref = xops.memory_efficient_attention_forward(
145
+ query.unsqueeze(0),
146
+ key.unsqueeze(0),
147
+ value.unsqueeze(0),
148
+ attn_bias=attn_bias,
149
+ p=0.0,
150
+ scale=scale,
151
+ op=attn_op,
152
+ )
153
+ torch.cuda.synchronize()
154
+ start_time = time.time()
155
+ output_ref = xops.memory_efficient_attention_forward(
156
+ query.unsqueeze(0),
157
+ key.unsqueeze(0),
158
+ value.unsqueeze(0),
159
+ attn_bias=attn_bias,
160
+ p=0.0,
161
+ scale=scale,
162
+ op=attn_op,
163
+ )
164
+ torch.cuda.synchronize()
165
+ end_time = time.time()
166
+ print(f"xformers Time: {(end_time - start_time)*1000:.2f} ms")
167
+ output_ref = output_ref.squeeze(0)
168
+ assert torch.allclose(output_ref, output, atol=1e-6, rtol=0)
tests/prefix_caching/test_prefix_caching.py
new
+41
@@ -0,0 +1,41 @@
1
+"""Compare the with and without prefix caching.
2
+
3
+Run `pytest tests/prefix_caching/test_prefix_caching.py`.
4
+"""
5
+import pytest
6
+
7
+from vllm import LLM, SamplingParams
8
+
9
+prefix = (
10
+ "You are an expert school principal, skilled in effectively managing "
11
+ "faculty and staff. Draft 10-15 questions for a potential first grade "
12
+ "Head Teacher for my K-12, all-girls', independent school that emphasizes "
13
+ "community, joyful discovery, and life-long learning. The candidate is "
14
+ "coming in for a first-round panel interview for a 8th grade Math "
15
+ "teaching role. They have 5 years of previous teaching experience "
16
+ "as an assistant teacher at a co-ed, public school with experience "
17
+ "in middle school math teaching. Based on these information, fulfill "
18
+ "the following paragraph: ")
19
+
20
+
21
+@pytest.mark.parametrize("model", ["facebook/opt-125m"])
22
+@pytest.mark.parametrize("max_tokens", [16])
23
+def test_prefix_caching(
24
+ example_prompts,
25
+ model: str,
26
+ max_tokens: int,
27
+):
28
+ llm = LLM(model=model)
29
+ # -1 since the last token can change when concatenating prompts.
30
+ prefix_pos = len(llm.llm_engine.tokenizer.encode(prefix)) - 1
31
+ prompts = [prefix + prompt for prompt in example_prompts]
32
+ sampling_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
33
+ outputs_without_prefix = llm.generate(prompts, sampling_params)
34
+ outputs_with_prefix = llm.generate(prompts,
35
+ sampling_params,
36
+ prefix_pos=[prefix_pos] * len(prompts))
37
+ for output_without_prefix, output_with_prefix in zip(
38
+ outputs_without_prefix, outputs_with_prefix):
39
+ assert (output_without_prefix.outputs[0].token_ids ==
40
+ output_with_prefix.outputs[0].token_ids)
41
+ assert len(llm.llm_engine.scheduler.prefix_pool.prefixes) == 1
tests/samplers/test_sampler.py
+12
-6
@@ -66,7 +66,8 @@ def test_sampler_all_greedy(seed: int):
66
prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
67
68
sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
69
- prompt_lens)
69
+ prompt_lens,
70
+ subquery_lens=prompt_lens)
71
sampler_output = sampler(embedding=None,
72
hidden_states=input_tensor,
73
sampling_metadata=sampling_metadata)
@@ -105,7 +106,8 @@ def test_sampler_all_random(seed: int):
106
prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
107
108
sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
108
- prompt_lens)
109
+ prompt_lens,
110
+ subquery_lens=prompt_lens)
111
sampler_output = sampler(embedding=None,
112
hidden_states=input_tensor,
113
sampling_metadata=sampling_metadata)
@@ -140,7 +142,8 @@ def test_sampler_all_beam(seed: int):
142
prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
143
144
sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
143
- prompt_lens)
145
+ prompt_lens,
146
+ subquery_lens=prompt_lens)
147
sampler(embedding=None,
148
hidden_states=input_tensor,
149
sampling_metadata=sampling_metadata)
@@ -193,7 +196,8 @@ def test_sampler_mixed(seed: int):
196
prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
197
198
sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
196
- prompt_lens)
199
+ prompt_lens,
200
+ subquery_lens=prompt_lens)
201
sampler_output = sampler(embedding=None,
202
hidden_states=input_tensor,
203
sampling_metadata=sampling_metadata)
@@ -234,7 +238,8 @@ def test_sampler_logits_processors(seed: int):
238
prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
239
240
sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
237
- prompt_lens)
241
+ prompt_lens,
242
+ subquery_lens=prompt_lens)
243
sampler_output = sampler(embedding=None,
244
hidden_states=input_tensor,
245
sampling_metadata=sampling_metadata)
@@ -288,7 +293,8 @@ def test_sampler_top_k_top_p(seed: int):
293
prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
294
295
sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
291
- prompt_lens)
296
+ prompt_lens,
297
+ subquery_lens=prompt_lens)
298
299
sample_probs = None
300
tests/worker/test_model_runner.py
+3
-2
@@ -33,11 +33,12 @@ def test_prepare_prompt():
33
expected_selected_token_indices.append(selected_token_start_idx +
34
prompt_len - 1)
35
selected_token_start_idx += max_seq_len
36
- input_tokens, input_positions, _, return_prompt_lens = (
36
+ input_tokens, input_positions, _, return_prompt_lens, _ = (
37
model_runner._prepare_prompt(seq_group_metadata_list))
38
assert return_prompt_lens == prompt_lens
39
sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
40
- prompt_lens)
40
+ prompt_lens,
41
+ subquery_lens=prompt_lens)
42
assert input_tokens.shape == (batch_size, max_seq_len)
43
assert input_positions.shape == (batch_size, max_seq_len)
44
torch.testing.assert_close(input_tokens, input_positions)
vllm/block.py
+4
@@ -66,3 +66,7 @@ class PhysicalTokenBlock:
66
return (f'PhysicalTokenBlock(device={self.device}, '
67
f'block_number={self.block_number}, '
68
f'ref_count={self.ref_count})')
69
+
70
+
71
+# Mapping: logical block number -> physical block.
72
+BlockTable = List[PhysicalTokenBlock]
vllm/core/block_manager.py
+42
-5
@@ -2,13 +2,10 @@
2
import enum
3
from typing import Dict, List, Optional, Set, Tuple
4
5
-from vllm.block import PhysicalTokenBlock
5
+from vllm.block import BlockTable, PhysicalTokenBlock
6
from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
7
from vllm.utils import Device
8
9
-# Mapping: logical block number -> physical block.
10
-BlockTable = List[PhysicalTokenBlock]
11
-
9
10
class BlockAllocator:
11
"""Manages free physical token blocks for a device.
@@ -105,6 +102,10 @@ class BlockSpaceManager:
102
# the same prompt. This may not be true for preempted sequences.
103
seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
104
num_required_blocks = len(seq.logical_token_blocks)
105
+
106
+ if seq_group.prefix is not None and seq_group.prefix.allocated:
107
+ num_required_blocks -= seq_group.prefix.get_num_blocks()
108
+
109
if self.block_sliding_window is not None:
110
num_required_blocks = min(num_required_blocks,
111
self.block_sliding_window)
@@ -125,8 +126,21 @@ class BlockSpaceManager:
126
seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
127
128
# Allocate new physical token blocks that will store the prompt tokens.
129
+ num_prompt_blocks = len(seq.logical_token_blocks)
130
+
131
block_table: BlockTable = []
129
- for logical_idx in range(len(seq.logical_token_blocks)):
132
+ prefix_block_table: BlockTable = []
133
+ num_prefix_blocks = 0
134
+
135
+ prefix = seq_group.prefix
136
+ if prefix is not None and prefix.allocated:
137
+ # Prefix has already been allocated. Use the existing block table.
138
+ num_prompt_blocks -= prefix.get_num_blocks()
139
+ for block in prefix.block_table:
140
+ block.ref_count += seq_group.num_seqs()
141
+ block_table.append(block)
142
+
143
+ for logical_idx in range(num_prompt_blocks):
144
if (self.block_sliding_window is not None
145
and logical_idx >= self.block_sliding_window):
146
block = block_table[logical_idx % self.block_sliding_window]
@@ -136,6 +150,15 @@ class BlockSpaceManager:
150
block.ref_count = seq_group.num_seqs()
151
block_table.append(block)
152
153
+ if prefix is not None and not prefix.allocated:
154
+ # Allocate blocks for the prefix, we will compute the prefix's
155
+ # KV cache in this run.
156
+ num_prefix_blocks = prefix.get_num_blocks()
157
+ prefix_block_table = block_table[:num_prefix_blocks]
158
+ for block in prefix_block_table:
159
+ block.ref_count += 1
160
+ prefix.set_block_table(prefix_block_table)
161
+
162
# Assign the block table for each sequence.
163
for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
164
self.block_tables[seq.seq_id] = block_table.copy()
@@ -210,10 +233,18 @@ class BlockSpaceManager:
233
234
def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
235
# CPU block -> GPU block.
236
+ if seq_group.prefix is not None:
237
+ # make sure to swap in the prefix first
238
+ assert seq_group.prefix.allocated and seq_group.prefix.computed
239
+
240
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
241
for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
242
new_block_table: BlockTable = []
243
block_table = self.block_tables[seq.seq_id]
244
+ if seq_group.prefix is not None:
245
+ for block in seq_group.prefix.block_table:
246
+ new_block_table.append(block)
247
+ block.ref_count += 1
248
249
for cpu_block in block_table:
250
if cpu_block in mapping:
@@ -245,6 +276,12 @@ class BlockSpaceManager:
276
block_table = self.block_tables[seq.seq_id]
277
278
for gpu_block in block_table:
279
+ if (seq_group.prefix is not None
280
+ and gpu_block in seq_group.prefix.block_table):
281
+ # NOTE: We do not swap out the prefix blocks for now.
282
+ self.gpu_allocator.free(gpu_block)
283
+ continue
284
+
285
if gpu_block in mapping:
286
cpu_block = mapping[gpu_block]
287
cpu_block.ref_count += 1
vllm/core/scheduler.py
+5
@@ -9,6 +9,7 @@ from vllm.core.policy import PolicyFactory
9
from vllm.logger import init_logger
10
from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
11
SequenceGroupMetadata, SequenceStatus)
12
+from vllm.prefix import PrefixPool
13
14
logger = init_logger(__name__)
15
@@ -76,6 +77,9 @@ class Scheduler:
77
num_cpu_blocks=self.cache_config.num_cpu_blocks,
78
sliding_window=self.cache_config.sliding_window)
79
80
+ # Create the prefix pool to cache the prefixes.
81
+ self.prefix_pool = PrefixPool(self.cache_config.block_size)
82
+
83
# Sequence groups in the WAITING state.
84
self.waiting: Deque[SequenceGroup] = deque()
85
# Sequence groups in the RUNNING state.
@@ -316,6 +320,7 @@ class Scheduler:
320
seq_data=seq_data,
321
sampling_params=seq_group.sampling_params,
322
block_tables=block_tables,
323
+ prefix=seq_group.prefix,
324
)
325
seq_group_metadata_list.append(seq_group_metadata)
326
return seq_group_metadata_list, scheduler_outputs
vllm/engine/async_llm_engine.py
+13
-3
@@ -371,6 +371,7 @@ class AsyncLLMEngine:
371
sampling_params: SamplingParams,
372
prompt_token_ids: Optional[List[int]] = None,
373
arrival_time: Optional[float] = None,
374
+ prefix_pos: Optional[int] = None,
375
) -> AsyncStream:
376
if self.log_requests:
377
shortened_prompt = prompt
@@ -383,6 +384,7 @@ class AsyncLLMEngine:
384
max_log_len]
385
logger.info(f"Received request {request_id}: "
386
f"prompt: {shortened_prompt!r}, "
387
+ f"prefix_pos: {prefix_pos},"
388
f"sampling params: {sampling_params}, "
389
f"prompt token ids: {shortened_token_ids}.")
390
@@ -401,7 +403,8 @@ class AsyncLLMEngine:
403
prompt=prompt,
404
sampling_params=sampling_params,
405
prompt_token_ids=prompt_token_ids,
404
- arrival_time=arrival_time)
406
+ arrival_time=arrival_time,
407
+ prefix_pos=prefix_pos)
408
409
return stream
410
@@ -410,7 +413,8 @@ class AsyncLLMEngine:
413
prompt: Optional[str],
414
sampling_params: SamplingParams,
415
request_id: str,
413
- prompt_token_ids: Optional[List[int]] = None
416
+ prompt_token_ids: Optional[List[int]] = None,
417
+ prefix_pos: Optional[int] = None,
418
) -> AsyncIterator[RequestOutput]:
419
"""Generate outputs for a request.
420
@@ -425,6 +429,11 @@ class AsyncLLMEngine:
429
request_id: The unique id of the request.
430
prompt_token_ids: The token IDs of the prompt. If None, we
431
use the tokenizer to convert the prompts to token IDs.
432
+ prefix_pos: If not None, we use the given position as the prefix
433
+ position for each prompt. We will cache the prefix's KV
434
+ cache and reuse it for the next request with the same prefix.
435
+ This is an experimental feature, and may be replaced with
436
+ automatic prefix caching in the future.
437
438
Yields:
439
The output `RequestOutput` objects from the LLMEngine for the
@@ -482,7 +491,8 @@ class AsyncLLMEngine:
491
prompt,
492
sampling_params,
493
prompt_token_ids=prompt_token_ids,
485
- arrival_time=arrival_time)
494
+ arrival_time=arrival_time,
495
+ prefix_pos=prefix_pos)
496
497
async for request_output in stream:
498
yield request_output
vllm/engine/llm_engine.py
+17
-1
@@ -337,6 +337,7 @@ class LLMEngine:
337
sampling_params: SamplingParams,
338
prompt_token_ids: Optional[List[int]] = None,
339
arrival_time: Optional[float] = None,
340
+ prefix_pos: Optional[int] = None,
341
) -> None:
342
"""Add a request to the engine's request pool.
343
@@ -353,6 +354,11 @@ class LLMEngine:
354
use the tokenizer to convert the prompts to token IDs.
355
arrival_time: The arrival time of the request. If None, we use
356
the current monotonic time.
357
+ prefix_pos: If not None, we use the given position as the prefix
358
+ position for each prompt. We will cache the prefix's KV
359
+ cache and reuse it for the next request with the same prefix.
360
+ This is an experimental feature, and may be replaced with
361
+ automatic prefix caching in the future.
362
363
Details:
364
- Set arrival_time to the current time if it is None.
@@ -389,9 +395,13 @@ class LLMEngine:
395
seq_id = next(self.seq_counter)
396
seq = Sequence(seq_id, prompt, prompt_token_ids, block_size)
397
398
+ # Check whether the input specifies prefix
399
+ prefix = self.scheduler.prefix_pool.add_or_get_prefix(
400
+ prompt_token_ids[:prefix_pos]) if prefix_pos is not None else None
401
+
402
# Create the sequence group.
403
seq_group = SequenceGroup(request_id, [seq], sampling_params,
394
- arrival_time)
404
+ arrival_time, prefix)
405
406
# Add the sequence group to the scheduler.
407
self.scheduler.add_seq_group(seq_group)
@@ -662,6 +672,12 @@ class LLMEngine:
672
request_output = RequestOutput.from_seq_group(seq_group)
673
request_outputs.append(request_output)
674
675
+ # Update prefix state, now all the uncomputed prefixes are computed.
676
+ for seq_group in scheduled_seq_groups:
677
+ if (seq_group.prefix is not None and seq_group.prefix.allocated
678
+ and not seq_group.prefix.computed):
679
+ seq_group.prefix.computed = True
680
+
681
if self.log_stats:
682
# Log the system stats.
683
self._log_system_stats(scheduler_outputs.prompt_run,
vllm/entrypoints/api_server.py
+5
-1
@@ -33,11 +33,15 @@ async def generate(request: Request) -> Response:
33
"""
34
request_dict = await request.json()
35
prompt = request_dict.pop("prompt")
36
+ prefix_pos = request_dict.pop("prefix_pos", None)
37
stream = request_dict.pop("stream", False)
38
sampling_params = SamplingParams(**request_dict)
39
request_id = random_uuid()
40
40
- results_generator = engine.generate(prompt, sampling_params, request_id)
41
+ results_generator = engine.generate(prompt,
42
+ sampling_params,
43
+ request_id,
44
+ prefix_pos=prefix_pos)
45
46
# Streaming case
47
async def stream_results() -> AsyncGenerator[bytes, None]:
vllm/entrypoints/llm.py
+14
-3
@@ -120,6 +120,7 @@ class LLM:
120
prompts: Optional[Union[str, List[str]]] = None,
121
sampling_params: Optional[SamplingParams] = None,
122
prompt_token_ids: Optional[List[List[int]]] = None,
123
+ prefix_pos: Optional[Union[int, List[int]]] = None,
124
use_tqdm: bool = True,
125
) -> List[RequestOutput]:
126
"""Generates the completions for the input prompts.
@@ -134,6 +135,11 @@ class LLM:
135
None, we use the default sampling parameters.
136
prompt_token_ids: A list of token IDs for the prompts. If None, we
137
use the tokenizer to convert the prompts to token IDs.
138
+ prefix_pos: If not None, we use the given position as the prefix
139
+ position for each prompt. We will cache the prefix's KV
140
+ cache and reuse it for the next request with the same prefix.
141
+ This is an experimental feature, and may be replaced with
142
+ automatic prefix caching in the future.
143
use_tqdm: Whether to use tqdm to display the progress bar.
144
145
Returns:
@@ -159,9 +165,10 @@ class LLM:
165
prompt_token_ids)
166
for i in range(num_requests):
167
prompt = prompts[i] if prompts is not None else None
168
+ prefix_pos_i = prefix_pos[i] if prefix_pos is not None else None
169
token_ids = None if prompt_token_ids is None else prompt_token_ids[
170
i]
164
- self._add_request(prompt, sampling_params, token_ids)
171
+ self._add_request(prompt, sampling_params, token_ids, prefix_pos_i)
172
return self._run_engine(use_tqdm)
173
174
def _add_request(
@@ -169,10 +176,14 @@ class LLM:
176
prompt: Optional[str],
177
sampling_params: SamplingParams,
178
prompt_token_ids: Optional[List[int]],
179
+ prefix_pos: Optional[int] = None,
180
) -> None:
181
request_id = str(next(self.request_counter))
174
- self.llm_engine.add_request(request_id, prompt, sampling_params,
175
- prompt_token_ids)
182
+ self.llm_engine.add_request(request_id,
183
+ prompt,
184
+ sampling_params,
185
+ prompt_token_ids,
186
+ prefix_pos=prefix_pos)
187
188
def _run_engine(self, use_tqdm: bool) -> List[RequestOutput]:
189
# Initialize tqdm.
vllm/model_executor/input_metadata.py
+6
@@ -18,12 +18,18 @@ class InputMetadata:
18
self,
19
is_prompt: bool,
20
slot_mapping: torch.Tensor,
21
+ prompt_lens: Optional[torch.Tensor],
22
+ max_seq_len: Optional[int],
23
+ start_loc: Optional[torch.Tensor],
24
max_context_len: Optional[int],
25
context_lens: Optional[torch.Tensor],
26
block_tables: Optional[torch.Tensor],
27
use_cuda_graph: bool,
28
) -> None:
29
self.is_prompt = is_prompt
30
+ self.prompt_lens = prompt_lens
31
+ self.max_seq_len = max_seq_len
32
+ self.start_loc = start_loc
33
self.max_context_len = max_context_len
34
self.slot_mapping = slot_mapping
35
self.context_lens = context_lens
vllm/model_executor/layers/attention.py
+55
-33
@@ -10,6 +10,8 @@ from xformers.ops.fmha.attn_bias import (BlockDiagonalCausalMask,
10
from vllm._C import ops
11
from vllm._C import cache_ops
12
from vllm.model_executor.input_metadata import InputMetadata
13
+from vllm.model_executor.layers.triton_kernel.prefix_prefill import (
14
+ context_attention_fwd)
15
from vllm.utils import is_hip
16
17
_SUPPORTED_HEAD_SIZES = [64, 80, 96, 112, 128, 256]
@@ -115,45 +117,65 @@ class PagedAttention(nn.Module):
117
self.num_kv_heads,
118
self.num_queries_per_kv,
119
value.shape[-1])
120
+ # normal attention
121
+ if (key_cache is None or value_cache is None
122
+ or input_metadata.block_tables.numel() == 0):
123
+ # Set attention bias if not provided. This typically happens at
124
+ # the very attention layer of every iteration.
125
+ # FIXME(woosuk): This is a hack.
126
+ if input_metadata.attn_bias is None:
127
+ if self.alibi_slopes is None:
128
+ attn_bias = BlockDiagonalCausalMask.from_seqlens(
129
+ [seq_len] * batch_size)
130
+ if self.sliding_window is not None:
131
+ attn_bias = attn_bias.make_local_attention(
132
+ self.sliding_window)
133
+ input_metadata.attn_bias = attn_bias
134
+ else:
135
+ input_metadata.attn_bias = _make_alibi_bias(
136
+ self.alibi_slopes, self.num_kv_heads, batch_size,
137
+ seq_len, query.dtype)
138
119
- # Set attention bias if not provided. This typically happens at the
120
- # very attention layer of every iteration.
121
- # FIXME(woosuk): This is a hack.
122
- if input_metadata.attn_bias is None:
139
+ # TODO(woosuk): Too many view operations. Let's try to reduce
140
+ # them in the future for code readability.
141
if self.alibi_slopes is None:
124
- attn_bias = BlockDiagonalCausalMask.from_seqlens(
125
- [seq_len] * batch_size)
126
- if self.sliding_window is not None:
127
- attn_bias = attn_bias.make_local_attention(
128
- self.sliding_window)
129
- input_metadata.attn_bias = attn_bias
142
+ query = query.unsqueeze(0)
143
+ key = key.unsqueeze(0)
144
+ value = value.unsqueeze(0)
145
else:
131
- input_metadata.attn_bias = _make_alibi_bias(
132
- self.alibi_slopes, self.num_kv_heads, batch_size,
133
- seq_len, query.dtype)
146
+ query = query.unflatten(0, (batch_size, seq_len))
147
+ key = key.unflatten(0, (batch_size, seq_len))
148
+ value = value.unflatten(0, (batch_size, seq_len))
149
135
- # TODO(woosuk): Too many view operations. Let's try to reduce them
136
- # in the future for code readability.
137
- if self.alibi_slopes is None:
138
- query = query.unsqueeze(0)
139
- key = key.unsqueeze(0)
140
- value = value.unsqueeze(0)
150
+ out = xops.memory_efficient_attention_forward(
151
+ query,
152
+ key,
153
+ value,
154
+ attn_bias=input_metadata.attn_bias,
155
+ p=0.0,
156
+ scale=self.scale,
157
+ op=xops.fmha.MemoryEfficientAttentionFlashAttentionOp[0] if
158
+ (is_hip()) else None,
159
+ )
160
+ output = out.view_as(query)
161
else:
142
- query = query.unflatten(0, (batch_size, seq_len))
143
- key = key.unflatten(0, (batch_size, seq_len))
144
- value = value.unflatten(0, (batch_size, seq_len))
162
+ # prefix-enabled attention
163
+ output = torch.empty_like(query)
164
+ context_attention_fwd(
165
+ query,
166
+ key,
167
+ value,
168
+ output,
169
+ key_cache,
170
+ value_cache,
171
+ input_metadata.block_tables, # [BS, max_block_per_request]
172
+ input_metadata.start_loc,
173
+ input_metadata.prompt_lens,
174
+ input_metadata.context_lens,
175
+ input_metadata.max_seq_len,
176
+ getattr(self, "alibi_slopes", None),
177
+ )
178
146
- out = xops.memory_efficient_attention_forward(
147
- query,
148
- key,
149
- value,
150
- attn_bias=input_metadata.attn_bias,
151
- p=0.0,
152
- scale=self.scale,
153
- op=xops.fmha.MemoryEfficientAttentionFlashAttentionOp[0] if
154
- (is_hip()) else None,
155
- )
156
- output = out.view_as(query)
179
else:
180
# Decoding run.
181
output = _paged_attention(
vllm/model_executor/layers/triton_kernel/__init__.py
vllm/model_executor/layers/triton_kernel/prefix_prefill.py
new
+728
@@ -0,0 +1,728 @@
1
+# The kernels in this file are adapted from LightLLM's context_attention_fwd:
2
+# https://github.com/ModelTC/lightllm/blob/main/lightllm/models/llama/triton_kernel/context_flashattention_nopad.py
3
+
4
+import torch
5
+import triton
6
+import triton.language as tl
7
+
8
+if triton.__version__ >= "2.1.0":
9
+
10
+ @triton.jit
11
+ def _fwd_kernel(
12
+ Q,
13
+ K,
14
+ V,
15
+ K_cache,
16
+ V_cache,
17
+ B_Loc,
18
+ sm_scale,
19
+ B_Start_Loc,
20
+ B_Seqlen,
21
+ B_Ctxlen,
22
+ block_size,
23
+ x,
24
+ Out,
25
+ stride_b_loc_b,
26
+ stride_b_loc_s,
27
+ stride_qbs,
28
+ stride_qh,
29
+ stride_qd,
30
+ stride_kbs,
31
+ stride_kh,
32
+ stride_kd,
33
+ stride_vbs,
34
+ stride_vh,
35
+ stride_vd,
36
+ stride_obs,
37
+ stride_oh,
38
+ stride_od,
39
+ stride_k_cache_bs,
40
+ stride_k_cache_h,
41
+ stride_k_cache_d,
42
+ stride_k_cache_bl,
43
+ stride_k_cache_x,
44
+ stride_v_cache_bs,
45
+ stride_v_cache_h,
46
+ stride_v_cache_d,
47
+ stride_v_cache_bl,
48
+ BLOCK_M: tl.constexpr,
49
+ BLOCK_DMODEL: tl.constexpr,
50
+ BLOCK_N: tl.constexpr,
51
+ ):
52
+ cur_batch = tl.program_id(0)
53
+ cur_head = tl.program_id(1)
54
+ start_m = tl.program_id(2)
55
+
56
+ cur_batch_ctx_len = tl.load(B_Ctxlen + cur_batch)
57
+ cur_batch_seq_len = tl.load(B_Seqlen + cur_batch)
58
+ cur_batch_in_all_start_index = tl.load(B_Start_Loc + cur_batch)
59
+
60
+ block_start_loc = BLOCK_M * start_m
61
+
62
+ # initialize offsets
63
+ offs_n = tl.arange(0, BLOCK_N)
64
+ offs_d = tl.arange(0, BLOCK_DMODEL)
65
+ offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
66
+ off_q = (
67
+ (cur_batch_in_all_start_index + offs_m[:, None]) * stride_qbs +
68
+ cur_head * stride_qh + offs_d[None, :] * stride_qd)
69
+
70
+ q = tl.load(
71
+ Q + off_q,
72
+ mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len,
73
+ other=0.0)
74
+
75
+ # # initialize pointer to m and l
76
+ m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf")
77
+ l_i = tl.zeros([BLOCK_M], dtype=tl.float32)
78
+ acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32)
79
+
80
+ for start_n in range(0, cur_batch_ctx_len, BLOCK_N):
81
+ start_n = tl.multiple_of(start_n, BLOCK_N)
82
+ # -- compute qk ----
83
+ bn = tl.load(B_Loc + cur_batch * stride_b_loc_b +
84
+ ((start_n + offs_n) // block_size) * stride_b_loc_s,
85
+ mask=(start_n + offs_n) < cur_batch_ctx_len,
86
+ other=0)
87
+ off_k = (bn[None, :] * stride_k_cache_bs +
88
+ cur_head * stride_k_cache_h +
89
+ (offs_d[:, None] // x) * stride_k_cache_d +
90
+ ((start_n + offs_n[None, :]) % block_size) *
91
+ stride_k_cache_bl +
92
+ (offs_d[:, None] % x) * stride_k_cache_x)
93
+ off_v = (
94
+ bn[:, None] * stride_v_cache_bs + cur_head * stride_v_cache_h +
95
+ offs_d[None, :] * stride_v_cache_d +
96
+ (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl)
97
+ k = tl.load(K_cache + off_k,
98
+ mask=(start_n + offs_n[None, :]) < cur_batch_ctx_len,
99
+ other=0.0)
100
+
101
+ qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
102
+ qk += tl.dot(q, k)
103
+ qk = tl.where((start_n + offs_n[None, :]) < cur_batch_ctx_len, qk,
104
+ float("-inf"))
105
+ qk *= sm_scale
106
+
107
+ # -- compute m_ij, p, l_ij
108
+ m_ij = tl.max(qk, 1)
109
+ p = tl.exp(qk - m_ij[:, None])
110
+ l_ij = tl.sum(p, 1)
111
+ # -- update m_i and l_i
112
+ m_i_new = tl.maximum(m_i, m_ij)
113
+ alpha = tl.exp(m_i - m_i_new)
114
+ beta = tl.exp(m_ij - m_i_new)
115
+ l_i_new = alpha * l_i + beta * l_ij
116
+ # -- update output accumulator --
117
+ # scale p
118
+ p_scale = beta / l_i_new
119
+ p = p * p_scale[:, None]
120
+ # scale acc
121
+ acc_scale = l_i / l_i_new * alpha
122
+ acc = acc * acc_scale[:, None]
123
+ # update acc
124
+ v = tl.load(V_cache + off_v,
125
+ mask=(start_n + offs_n[:, None]) < cur_batch_ctx_len,
126
+ other=0.0)
127
+
128
+ p = p.to(v.dtype)
129
+ acc += tl.dot(p, v)
130
+ # # update m_i and l_i
131
+ l_i = l_i_new
132
+ m_i = m_i_new
133
+
134
+ off_k = (offs_n[None, :] * stride_kbs + cur_head * stride_kh +
135
+ offs_d[:, None] * stride_kd)
136
+ off_v = (offs_n[:, None] * stride_vbs + cur_head * stride_vh +
137
+ offs_d[None, :] * stride_vd)
138
+ k_ptrs = K + off_k
139
+ v_ptrs = V + off_v
140
+
141
+ block_mask = tl.where(
142
+ block_start_loc < cur_batch_seq_len - cur_batch_ctx_len, 1, 0)
143
+
144
+ for start_n in range(0, block_mask * (start_m + 1) * BLOCK_M, BLOCK_N):
145
+ start_n = tl.multiple_of(start_n, BLOCK_N)
146
+ # -- compute qk ----
147
+ k = tl.load(k_ptrs +
148
+ (cur_batch_in_all_start_index + start_n) * stride_kbs,
149
+ mask=(start_n + offs_n[None, :]) <
150
+ cur_batch_seq_len - cur_batch_ctx_len,
151
+ other=0.0)
152
+
153
+ qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
154
+ qk += tl.dot(q, k)
155
+ qk *= sm_scale
156
+ qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk,
157
+ float("-inf"))
158
+
159
+ # -- compute m_ij, p, l_ij
160
+ m_ij = tl.max(qk, 1)
161
+ p = tl.exp(qk - m_ij[:, None])
162
+ l_ij = tl.sum(p, 1)
163
+ # -- update m_i and l_i
164
+ m_i_new = tl.maximum(m_i, m_ij)
165
+ alpha = tl.exp(m_i - m_i_new)
166
+ beta = tl.exp(m_ij - m_i_new)
167
+ l_i_new = alpha * l_i + beta * l_ij
168
+ # -- update output accumulator --
169
+ # scale p
170
+ p_scale = beta / l_i_new
171
+ p = p * p_scale[:, None]
172
+ # scale acc
173
+ acc_scale = l_i / l_i_new * alpha
174
+ acc = acc * acc_scale[:, None]
175
+ # update acc
176
+ v = tl.load(v_ptrs +
177
+ (cur_batch_in_all_start_index + start_n) * stride_vbs,
178
+ mask=(start_n + offs_n[:, None]) <
179
+ cur_batch_seq_len - cur_batch_ctx_len,
180
+ other=0.0)
181
+
182
+ p = p.to(v.dtype)
183
+ acc += tl.dot(p, v)
184
+ # update m_i and l_i
185
+ l_i = l_i_new
186
+ m_i = m_i_new
187
+ # initialize pointers to output
188
+ off_o = (
189
+ (cur_batch_in_all_start_index + offs_m[:, None]) * stride_obs +
190
+ cur_head * stride_oh + offs_d[None, :] * stride_od)
191
+ out_ptrs = Out + off_o
192
+ tl.store(out_ptrs,
193
+ acc,
194
+ mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len)
195
+ return
196
+
197
+ @triton.jit
198
+ def _fwd_kernel_flash_attn_v2(
199
+ Q,
200
+ K,
201
+ V,
202
+ K_cache,
203
+ V_cache,
204
+ B_Loc,
205
+ sm_scale,
206
+ B_Start_Loc,
207
+ B_Seqlen,
208
+ B_Ctxlen,
209
+ block_size,
210
+ x,
211
+ Out,
212
+ stride_b_loc_b,
213
+ stride_b_loc_s,
214
+ stride_qbs,
215
+ stride_qh,
216
+ stride_qd,
217
+ stride_kbs,
218
+ stride_kh,
219
+ stride_kd,
220
+ stride_vbs,
221
+ stride_vh,
222
+ stride_vd,
223
+ stride_obs,
224
+ stride_oh,
225
+ stride_od,
226
+ stride_k_cache_bs,
227
+ stride_k_cache_h,
228
+ stride_k_cache_d,
229
+ stride_k_cache_bl,
230
+ stride_k_cache_x,
231
+ stride_v_cache_bs,
232
+ stride_v_cache_h,
233
+ stride_v_cache_d,
234
+ stride_v_cache_bl,
235
+ BLOCK_M: tl.constexpr,
236
+ BLOCK_DMODEL: tl.constexpr,
237
+ BLOCK_N: tl.constexpr,
238
+ ):
239
+ cur_batch = tl.program_id(0)
240
+ cur_head = tl.program_id(1)
241
+ start_m = tl.program_id(2)
242
+
243
+ cur_batch_ctx_len = tl.load(B_Ctxlen + cur_batch)
244
+ cur_batch_seq_len = tl.load(B_Seqlen + cur_batch)
245
+ cur_batch_in_all_start_index = tl.load(B_Start_Loc + cur_batch)
246
+
247
+ block_start_loc = BLOCK_M * start_m
248
+
249
+ # initialize offsets
250
+ offs_n = tl.arange(0, BLOCK_N)
251
+ offs_d = tl.arange(0, BLOCK_DMODEL)
252
+ offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
253
+ off_q = (
254
+ (cur_batch_in_all_start_index + offs_m[:, None]) * stride_qbs +
255
+ cur_head * stride_qh + offs_d[None, :] * stride_qd)
256
+
257
+ q = tl.load(
258
+ Q + off_q,
259
+ mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len,
260
+ other=0.0)
261
+
262
+ # # initialize pointer to m and l
263
+ m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf")
264
+ l_i = tl.zeros([BLOCK_M], dtype=tl.float32)
265
+ acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32)
266
+
267
+ for start_n in range(0, cur_batch_ctx_len, BLOCK_N):
268
+ start_n = tl.multiple_of(start_n, BLOCK_N)
269
+ # -- compute qk ----
270
+ bn = tl.load(B_Loc + cur_batch * stride_b_loc_b +
271
+ ((start_n + offs_n) // block_size) * stride_b_loc_s,
272
+ mask=(start_n + offs_n) < cur_batch_ctx_len,
273
+ other=0)
274
+ off_k = (bn[None, :] * stride_k_cache_bs +
275
+ cur_head * stride_k_cache_h +
276
+ (offs_d[:, None] // x) * stride_k_cache_d +
277
+ ((start_n + offs_n[None, :]) % block_size) *
278
+ stride_k_cache_bl +
279
+ (offs_d[:, None] % x) * stride_k_cache_x)
280
+ off_v = (
281
+ bn[:, None] * stride_v_cache_bs + cur_head * stride_v_cache_h +
282
+ offs_d[None, :] * stride_v_cache_d +
283
+ (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl)
284
+ k = tl.load(K_cache + off_k,
285
+ mask=(start_n + offs_n[None, :]) < cur_batch_ctx_len,
286
+ other=0.0)
287
+
288
+ qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
289
+ qk += tl.dot(q, k)
290
+ qk = tl.where((start_n + offs_n[None, :]) < cur_batch_ctx_len, qk,
291
+ float("-inf"))
292
+ qk *= sm_scale
293
+
294
+ # -- compute m_ij, p, l_ij
295
+ m_ij = tl.max(qk, 1)
296
+ m_i_new = tl.maximum(m_i, m_ij)
297
+ p = tl.math.exp(qk - m_i_new[:, None])
298
+ l_ij = tl.sum(p, 1)
299
+ # -- update m_i and l_i
300
+
301
+ alpha = tl.math.exp(m_i - m_i_new)
302
+ l_i_new = alpha * l_i + l_ij
303
+ # -- update output accumulator --
304
+ # scale p
305
+ # scale acc
306
+ acc_scale = alpha
307
+ # acc_scale = l_i / l_i_new * alpha
308
+ acc = acc * acc_scale[:, None]
309
+ # update acc
310
+ v = tl.load(V_cache + off_v,
311
+ mask=(start_n + offs_n[:, None]) < cur_batch_ctx_len,
312
+ other=0.0)
313
+
314
+ p = p.to(v.dtype)
315
+ acc += tl.dot(p, v)
316
+ # update m_i and l_i
317
+ l_i = l_i_new
318
+ m_i = m_i_new
319
+
320
+ off_k = (offs_n[None, :] * stride_kbs + cur_head * stride_kh +
321
+ offs_d[:, None] * stride_kd)
322
+ off_v = (offs_n[:, None] * stride_vbs + cur_head * stride_vh +
323
+ offs_d[None, :] * stride_vd)
324
+ k_ptrs = K + off_k
325
+ v_ptrs = V + off_v
326
+
327
+ block_mask = tl.where(
328
+ block_start_loc < cur_batch_seq_len - cur_batch_ctx_len, 1, 0)
329
+
330
+ for start_n in range(0, block_mask * (start_m + 1) * BLOCK_M, BLOCK_N):
331
+ start_n = tl.multiple_of(start_n, BLOCK_N)
332
+ # -- compute qk ----
333
+ k = tl.load(k_ptrs +
334
+ (cur_batch_in_all_start_index + start_n) * stride_kbs,
335
+ mask=(start_n + offs_n[None, :]) <
336
+ cur_batch_seq_len - cur_batch_ctx_len,
337
+ other=0.0)
338
+
339
+ qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
340
+ qk += tl.dot(q, k)
341
+ qk *= sm_scale
342
+ qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk,
343
+ float("-inf"))
344
+
345
+ # -- compute m_ij, p, l_ij
346
+ m_ij = tl.max(qk, 1)
347
+ m_i_new = tl.maximum(m_i, m_ij)
348
+ p = tl.math.exp(qk - m_i_new[:, None])
349
+ l_ij = tl.sum(p, 1)
350
+ # -- update m_i and l_i
351
+
352
+ alpha = tl.math.exp(m_i - m_i_new)
353
+ l_i_new = alpha * l_i + l_ij
354
+ # -- update output accumulator --
355
+ # scale p
356
+ # scale acc
357
+ acc_scale = alpha
358
+ # acc_scale = l_i / l_i_new * alpha
359
+ acc = acc * acc_scale[:, None]
360
+ # update acc
361
+ v = tl.load(v_ptrs +
362
+ (cur_batch_in_all_start_index + start_n) * stride_vbs,
363
+ mask=(start_n + offs_n[:, None]) <
364
+ cur_batch_seq_len - cur_batch_ctx_len,
365
+ other=0.0)
366
+
367
+ p = p.to(v.dtype)
368
+ acc += tl.dot(p, v)
369
+ # update m_i and l_i
370
+ l_i = l_i_new
371
+ m_i = m_i_new
372
+
373
+ # acc /= l_i[:, None]
374
+ # initialize pointers to output
375
+ off_o = (
376
+ (cur_batch_in_all_start_index + offs_m[:, None]) * stride_obs +
377
+ cur_head * stride_oh + offs_d[None, :] * stride_od)
378
+ out_ptrs = Out + off_o
379
+ tl.store(out_ptrs,
380
+ acc,
381
+ mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len)
382
+ return
383
+
384
+ @triton.jit
385
+ def _fwd_kernel_alibi(
386
+ Q,
387
+ K,
388
+ V,
389
+ K_cache,
390
+ V_cache,
391
+ B_Loc,
392
+ sm_scale,
393
+ B_Start_Loc,
394
+ B_Seqlen,
395
+ B_Ctxlen,
396
+ Alibi_slopes,
397
+ block_size,
398
+ x,
399
+ Out,
400
+ stride_b_loc_b,
401
+ stride_b_loc_s,
402
+ stride_qbs,
403
+ stride_qh,
404
+ stride_qd,
405
+ stride_kbs,
406
+ stride_kh,
407
+ stride_kd,
408
+ stride_vbs,
409
+ stride_vh,
410
+ stride_vd,
411
+ stride_obs,
412
+ stride_oh,
413
+ stride_od,
414
+ stride_k_cache_bs,
415
+ stride_k_cache_h,
416
+ stride_k_cache_d,
417
+ stride_k_cache_bl,
418
+ stride_k_cache_x,
419
+ stride_v_cache_bs,
420
+ stride_v_cache_h,
421
+ stride_v_cache_d,
422
+ stride_v_cache_bl,
423
+ BLOCK_M: tl.constexpr,
424
+ BLOCK_DMODEL: tl.constexpr,
425
+ BLOCK_N: tl.constexpr,
426
+ ):
427
+ # attn_bias[]
428
+ cur_batch = tl.program_id(0)
429
+ cur_head = tl.program_id(1)
430
+ start_m = tl.program_id(2)
431
+
432
+ # cur_batch_seq_len: the length of prompts
433
+ # cur_batch_ctx_len: the length of prefix
434
+ # cur_batch_in_all_start_index: the start id of the dim=0
435
+ cur_batch_ctx_len = tl.load(B_Ctxlen + cur_batch)
436
+ cur_batch_seq_len = tl.load(B_Seqlen + cur_batch)
437
+ cur_batch_in_all_start_index = tl.load(B_Start_Loc + cur_batch)
438
+
439
+ block_start_loc = BLOCK_M * start_m
440
+
441
+ # initialize offsets
442
+ offs_n = tl.arange(0, BLOCK_N)
443
+ offs_d = tl.arange(0, BLOCK_DMODEL)
444
+ offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
445
+ off_q = (
446
+ (cur_batch_in_all_start_index + offs_m[:, None]) * stride_qbs +
447
+ cur_head * stride_qh + offs_d[None, :] * stride_qd)
448
+
449
+ q = tl.load(
450
+ Q + off_q,
451
+ mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len,
452
+ other=0.0)
453
+
454
+ # # initialize pointer to m and l
455
+ m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf")
456
+ l_i = tl.zeros([BLOCK_M], dtype=tl.float32)
457
+ acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32)
458
+
459
+ alibi_slope = tl.load(Alibi_slopes + cur_head)
460
+ alibi_start_q = tl.arange(
461
+ 0, BLOCK_M) + block_start_loc + cur_batch_ctx_len
462
+ alibi_start_k = 0
463
+ for start_n in range(0, cur_batch_ctx_len, BLOCK_N):
464
+ start_n = tl.multiple_of(start_n, BLOCK_N)
465
+ # -- compute qk ----
466
+ bn = tl.load(B_Loc + cur_batch * stride_b_loc_b +
467
+ ((start_n + offs_n) // block_size) * stride_b_loc_s,
468
+ mask=(start_n + offs_n) < cur_batch_ctx_len,
469
+ other=0)
470
+ off_k = (bn[None, :] * stride_k_cache_bs +
471
+ cur_head * stride_k_cache_h +
472
+ (offs_d[:, None] // x) * stride_k_cache_d +
473
+ ((start_n + offs_n[None, :]) % block_size) *
474
+ stride_k_cache_bl +
475
+ (offs_d[:, None] % x) * stride_k_cache_x)
476
+ off_v = (
477
+ bn[:, None] * stride_v_cache_bs + cur_head * stride_v_cache_h +
478
+ offs_d[None, :] * stride_v_cache_d +
479
+ (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl)
480
+ k = tl.load(K_cache + off_k,
481
+ mask=(start_n + offs_n[None, :]) < cur_batch_ctx_len,
482
+ other=0.0)
483
+
484
+ qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
485
+ qk += tl.dot(q, k)
486
+ qk = tl.where((start_n + offs_n[None, :]) < cur_batch_ctx_len, qk,
487
+ float("-inf"))
488
+ qk *= sm_scale
489
+
490
+ # load alibi
491
+ alibi = (tl.arange(0, BLOCK_N)[None, :] + alibi_start_k -
492
+ alibi_start_q[:, None]) * alibi_slope
493
+ alibi = tl.where(
494
+ (alibi <= 0) & (alibi_start_q[:, None] < cur_batch_seq_len),
495
+ alibi, float("-inf"))
496
+ qk += alibi
497
+ alibi_start_k += BLOCK_N
498
+
499
+ # -- compute m_ij, p, l_ij
500
+ m_ij = tl.max(qk, 1)
501
+ m_i_new = tl.maximum(m_i, m_ij)
502
+ p = tl.math.exp(qk - m_i_new[:, None])
503
+ l_ij = tl.sum(p, 1)
504
+ # -- update m_i and l_i
505
+
506
+ alpha = tl.math.exp(m_i - m_i_new)
507
+ l_i_new = alpha * l_i + l_ij
508
+ # -- update output accumulator --
509
+ # scale p
510
+ # scale acc
511
+ acc_scale = alpha
512
+ # acc_scale = l_i / l_i_new * alpha
513
+ acc = acc * acc_scale[:, None]
514
+ # update acc
515
+ v = tl.load(V_cache + off_v,
516
+ mask=(start_n + offs_n[:, None]) < cur_batch_ctx_len,
517
+ other=0.0)
518
+
519
+ p = p.to(v.dtype)
520
+ acc += tl.dot(p, v, allow_tf32=False)
521
+ # update m_i and l_i
522
+ l_i = l_i_new
523
+ m_i = m_i_new
524
+
525
+ off_k = (offs_n[None, :] * stride_kbs + cur_head * stride_kh +
526
+ offs_d[:, None] * stride_kd)
527
+ off_v = (offs_n[:, None] * stride_vbs + cur_head * stride_vh +
528
+ offs_d[None, :] * stride_vd)
529
+ k_ptrs = K + off_k
530
+ v_ptrs = V + off_v
531
+
532
+ block_mask = tl.where(
533
+ block_start_loc < cur_batch_seq_len - cur_batch_ctx_len, 1, 0)
534
+
535
+ # init alibi
536
+ alibi_slope = tl.load(Alibi_slopes + cur_head)
537
+ alibi_start_q = tl.arange(
538
+ 0, BLOCK_M) + block_start_loc + cur_batch_ctx_len
539
+ alibi_start_k = cur_batch_ctx_len
540
+ # # init debuger
541
+ # offset_db_q = tl.arange(0, BLOCK_M) + block_start_loc
542
+ # offset_db_k = tl.arange(0, BLOCK_N)
543
+ # calc q[BLOCK_M, BLOCK_MODEL] mul k[prefix_len: , BLOCK_DMODEL]
544
+ for start_n in range(0, block_mask * (start_m + 1) * BLOCK_M, BLOCK_N):
545
+ start_n = tl.multiple_of(start_n, BLOCK_N)
546
+ # -- compute qk ----
547
+ k = tl.load(k_ptrs +
548
+ (cur_batch_in_all_start_index + start_n) * stride_kbs,
549
+ mask=(start_n + offs_n[None, :]) <
550
+ cur_batch_seq_len - cur_batch_ctx_len,
551
+ other=0.0)
552
+
553
+ qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
554
+ qk += tl.dot(q, k, allow_tf32=False)
555
+ qk *= sm_scale
556
+ qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk,
557
+ float("-inf"))
558
+
559
+ # load alibi
560
+ alibi = (tl.arange(0, BLOCK_N)[None, :] + alibi_start_k -
561
+ alibi_start_q[:, None]) * alibi_slope
562
+ alibi = tl.where(
563
+ (alibi <= 0) & (alibi_start_q[:, None] < cur_batch_seq_len),
564
+ alibi, float("-inf"))
565
+ qk += alibi
566
+ alibi_start_k += BLOCK_N
567
+
568
+ # -- compute m_ij, p, l_ij
569
+ m_ij = tl.max(qk, 1)
570
+ m_i_new = tl.maximum(m_i, m_ij)
571
+ p = tl.math.exp(qk - m_i_new[:, None])
572
+ l_ij = tl.sum(p, 1)
573
+ # -- update m_i and l_i
574
+
575
+ alpha = tl.math.exp(m_i - m_i_new)
576
+ l_i_new = alpha * l_i + l_ij
577
+ # -- update output accumulator --
578
+ # scale p
579
+ # scale acc
580
+ acc_scale = alpha
581
+ # acc_scale = l_i / l_i_new * alpha
582
+ acc = acc * acc_scale[:, None]
583
+ # update acc
584
+ v = tl.load(v_ptrs +
585
+ (cur_batch_in_all_start_index + start_n) * stride_vbs,
586
+ mask=(start_n + offs_n[:, None]) <
587
+ cur_batch_seq_len - cur_batch_ctx_len,
588
+ other=0.0)
589
+
590
+ p = p.to(v.dtype)
591
+ acc += tl.dot(p, v, allow_tf32=False)
592
+ # update m_i and l_i
593
+ l_i = l_i_new
594
+ m_i = m_i_new
595
+
596
+ acc = acc / l_i[:, None]
597
+
598
+ # initialize pointers to output
599
+ off_o = (
600
+ (cur_batch_in_all_start_index + offs_m[:, None]) * stride_obs +
601
+ cur_head * stride_oh + offs_d[None, :] * stride_od)
602
+ out_ptrs = Out + off_o
603
+ tl.store(out_ptrs,
604
+ acc,
605
+ mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len)
606
+ return
607
+
608
+ @torch.inference_mode()
609
+ def context_attention_fwd(q,
610
+ k,
611
+ v,
612
+ o,
613
+ k_cache,
614
+ v_cache,
615
+ b_loc,
616
+ b_start_loc,
617
+ b_seq_len,
618
+ b_ctx_len,
619
+ max_input_len,
620
+ alibi_slopes=None):
621
+ BLOCK = 128
622
+ # shape constraints
623
+ Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1]
624
+ assert Lq == Lk and Lk == Lv
625
+ assert Lk in {16, 32, 64, 128}
626
+
627
+ sm_scale = 1.0 / (Lq**0.5)
628
+ batch, head = b_seq_len.shape[0], q.shape[1]
629
+
630
+ grid = (batch, head, triton.cdiv(max_input_len, BLOCK)) # batch, head,
631
+
632
+ num_warps = 8 if Lk <= 64 else 8
633
+ if alibi_slopes is not None:
634
+ _fwd_kernel_alibi[grid](
635
+ q,
636
+ k,
637
+ v,
638
+ k_cache,
639
+ v_cache,
640
+ b_loc,
641
+ sm_scale,
642
+ b_start_loc,
643
+ b_seq_len,
644
+ b_ctx_len,
645
+ alibi_slopes,
646
+ v_cache.shape[3],
647
+ 8,
648
+ o,
649
+ b_loc.stride(0),
650
+ b_loc.stride(1),
651
+ q.stride(0),
652
+ q.stride(1),
653
+ q.stride(2),
654
+ k.stride(0),
655
+ k.stride(1),
656
+ k.stride(2),
657
+ v.stride(0),
658
+ v.stride(1),
659
+ v.stride(2),
660
+ o.stride(0),
661
+ o.stride(1),
662
+ o.stride(2),
663
+ k_cache.stride(0),
664
+ k_cache.stride(1),
665
+ k_cache.stride(2),
666
+ k_cache.stride(3),
667
+ k_cache.stride(
668
+ 4
669
+ ), #[num_blocks, num_kv_heads, head_size/x, block_size, x]
670
+ v_cache.stride(0),
671
+ v_cache.stride(1),
672
+ v_cache.stride(2),
673
+ v_cache.stride(
674
+ 3), #[num_blocks, num_kv_heads, head_size, block_size]
675
+ BLOCK_M=BLOCK,
676
+ BLOCK_DMODEL=Lk,
677
+ BLOCK_N=BLOCK,
678
+ num_warps=num_warps,
679
+ num_stages=1,
680
+ )
681
+ return
682
+
683
+ _fwd_kernel[grid](
684
+ q,
685
+ k,
686
+ v,
687
+ k_cache,
688
+ v_cache,
689
+ b_loc,
690
+ sm_scale,
691
+ b_start_loc,
692
+ b_seq_len,
693
+ b_ctx_len,
694
+ v_cache.shape[3],
695
+ 8,
696
+ o,
697
+ b_loc.stride(0),
698
+ b_loc.stride(1),
699
+ q.stride(0),
700
+ q.stride(1),
701
+ q.stride(2),
702
+ k.stride(0),
703
+ k.stride(1),
704
+ k.stride(2),
705
+ v.stride(0),
706
+ v.stride(1),
707
+ v.stride(2),
708
+ o.stride(0),
709
+ o.stride(1),
710
+ o.stride(2),
711
+ k_cache.stride(0),
712
+ k_cache.stride(1),
713
+ k_cache.stride(2),
714
+ k_cache.stride(3),
715
+ k_cache.stride(
716
+ 4), #[num_blocks, num_kv_heads, head_size/x, block_size, x]
717
+ v_cache.stride(0),
718
+ v_cache.stride(1),
719
+ v_cache.stride(2),
720
+ v_cache.stride(
721
+ 3), #[num_blocks, num_kv_heads, head_size, block_size]
722
+ BLOCK_M=BLOCK,
723
+ BLOCK_DMODEL=Lk,
724
+ BLOCK_N=BLOCK,
725
+ num_warps=num_warps,
726
+ num_stages=1,
727
+ )
728
+ return
vllm/prefix.py
new
+87
@@ -0,0 +1,87 @@
1
+from typing import Dict, List, Sequence, Tuple, Optional
2
+
3
+from vllm.block import BlockTable
4
+
5
+
6
+class Prefix:
7
+ """Data and states associated with a prefix of prompt tokens for multiple
8
+ sequence groups.
9
+
10
+ NOTE: This feature is experimental and may be replaced with automatic
11
+ prefix caching in the future.
12
+
13
+ Args:
14
+ prefix_id: The id of the prefix in the prefix pool.
15
+ token_ids: The token ids of the prefix.
16
+ block_size: The block size of the executed model.
17
+ """
18
+
19
+ def __init__(
20
+ self,
21
+ token_ids: Sequence[int],
22
+ block_size: int,
23
+ ) -> None:
24
+ self.token_ids = tuple(token_ids)
25
+ self.block_size = block_size
26
+ self.length = len(token_ids)
27
+ self.hash = hash(token_ids)
28
+ assert self.length % block_size == 0
29
+ self.block_table: Optional[BlockTable] = None
30
+ self.computed = False
31
+
32
+ @property
33
+ def allocated(self) -> bool:
34
+ return self.block_table is not None
35
+
36
+ def get_num_blocks(self) -> int:
37
+ return self.length // self.block_size
38
+
39
+ def get_block_numbers(self) -> List[int]:
40
+ return [block.block_number for block in self.block_table]
41
+
42
+ def get_length(self) -> int:
43
+ return self.length
44
+
45
+ def __hash__(self) -> int:
46
+ return self.hash
47
+
48
+ def set_block_table(self, block_table: BlockTable) -> None:
49
+ self.block_table = block_table.copy()
50
+
51
+
52
+class PrefixPool:
53
+ """Manages all the prompt prefixes.
54
+
55
+ NOTE: This feature is experimental and may be replaced with automatic
56
+ prefix caching in the future.
57
+
58
+ Args:
59
+ block_size: The block size of the executed model.
60
+
61
+ Attributes:
62
+ prefixes: A list of all the prefixes.
63
+ block_size: The block size of the executed model.
64
+ """
65
+
66
+ def __init__(
67
+ self,
68
+ block_size: int,
69
+ ) -> None:
70
+ # TODO(zhuohan): Add a capacity limit to the prefix pool.
71
+ self.prefixes: Dict[int, Prefix] = {}
72
+ self.block_size = block_size
73
+
74
+ def _truncate_token_ids(self, token_ids: Sequence[int]) -> Tuple[int]:
75
+ new_length = len(token_ids) // self.block_size * self.block_size
76
+ return tuple(token_ids[:new_length])
77
+
78
+ def add_or_get_prefix(self, token_ids: Sequence[int]) -> Optional[Prefix]:
79
+ token_ids = self._truncate_token_ids(token_ids)
80
+ if len(token_ids) == 0:
81
+ # Prefix is empty.
82
+ return None
83
+ prefix = Prefix(token_ids, self.block_size)
84
+ prefix_hash = hash(prefix)
85
+ if prefix_hash not in self.prefixes:
86
+ self.prefixes[prefix_hash] = prefix
87
+ return self.prefixes[prefix_hash]
vllm/sequence.py
+6
-1
@@ -4,6 +4,7 @@ import enum
4
from typing import Dict, List, Optional, Union
5
6
from vllm.block import LogicalTokenBlock
7
+from vllm.prefix import Prefix
8
from vllm.sampling_params import SamplingParams
9
10
PromptLogprobs = List[Optional[Dict[int, float]]]
@@ -236,11 +237,13 @@ class SequenceGroup:
237
seqs: List[Sequence],
238
sampling_params: SamplingParams,
239
arrival_time: float,
240
+ prefix: Optional[Prefix] = None,
241
) -> None:
242
self.request_id = request_id
243
self.seqs_dict = {seq.seq_id: seq for seq in seqs}
244
self.sampling_params = sampling_params
245
self.arrival_time = arrival_time
246
+ self.prefix: Optional[Prefix] = prefix
247
self.prompt_logprobs: Optional[PromptLogprobs] = None
248
249
@property
@@ -327,7 +330,6 @@ class SequenceGroup:
330
class SequenceGroupMetadata:
331
"""Metadata for a sequence group. Used to create `InputMetadata`.
332
330
-
333
Args:
334
request_id: The ID of the request.
335
is_prompt: Whether the request is at prompt stage.
@@ -335,6 +337,7 @@ class SequenceGroupMetadata:
337
sampling_params: The sampling parameters used to generate the outputs.
338
block_tables: The block tables. (Seq id -> list of physical block
339
numbers)
340
+ prefix: The prefix of the prompt of the sequence group.
341
"""
342
343
def __init__(
@@ -344,12 +347,14 @@ class SequenceGroupMetadata:
347
seq_data: Dict[int, SequenceData],
348
sampling_params: SamplingParams,
349
block_tables: Dict[int, List[int]],
350
+ prefix: Optional[Prefix] = None,
351
) -> None:
352
self.request_id = request_id
353
self.is_prompt = is_prompt
354
self.seq_data = seq_data
355
self.sampling_params = sampling_params
356
self.block_tables = block_tables
357
+ self.prefix = prefix
358
359
360
class SequenceOutput:
vllm/worker/model_runner.py
+95
-16
@@ -74,13 +74,17 @@ class ModelRunner:
74
def _prepare_prompt(
75
self,
76
seq_group_metadata_list: List[SequenceGroupMetadata],
77
- ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, List[int]]:
77
+ ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, List[int],
78
+ List[int]]:
79
assert len(seq_group_metadata_list) > 0
80
input_tokens: List[List[int]] = []
81
input_positions: List[List[int]] = []
82
slot_mapping: List[List[int]] = []
83
84
prompt_lens: List[int] = []
85
+ context_lens: List[int] = []
86
+ subquery_lens: List[int] = []
87
+ prefix_block_tables: List[List[int]] = []
88
for seq_group_metadata in seq_group_metadata_list:
89
assert seq_group_metadata.is_prompt
90
seq_ids = list(seq_group_metadata.seq_data.keys())
@@ -91,11 +95,23 @@ class ModelRunner:
95
prompt_tokens = seq_data.get_token_ids()
96
prompt_len = len(prompt_tokens)
97
prompt_lens.append(prompt_len)
98
+ prefix_len = 0
99
+ prefix = seq_group_metadata.prefix
100
+ if prefix is not None and prefix.computed:
101
+ prefix_len = prefix.get_length()
102
+ prompt_tokens = prompt_tokens[prefix_len:]
103
+ prefix_block_tables.append(prefix.get_block_numbers())
104
+ else:
105
+ prefix_block_tables.append([])
106
+ # actual prompt lens
107
+ context_lens.append(prefix_len)
108
+ subquery_lens.append(prompt_len - prefix_len)
109
110
input_tokens.append(prompt_tokens)
111
# NOTE(woosuk): Here we assume that the first token in the prompt
112
# is always the first token in the sequence.
98
- input_positions.append(list(range(prompt_len)))
113
+ input_positions.append(
114
+ list(range(prefix_len, prefix_len + len(prompt_tokens))))
115
116
if seq_group_metadata.block_tables is None:
117
# During memory profiling, the block tables are not initialized
@@ -113,8 +129,11 @@ class ModelRunner:
129
# mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
130
start_idx = 0
131
if self.sliding_window is not None:
132
+ assert prefix_len == 0, (
133
+ "Prefix caching is currently not supported with "
134
+ "sliding window attention")
135
start_idx = max(0, prompt_len - self.sliding_window)
117
- for i in range(prompt_len):
136
+ for i in range(prefix_len, prompt_len):
137
if i < start_idx:
138
slot_mapping[-1].append(_PAD_SLOT_ID)
139
continue
@@ -124,7 +143,7 @@ class ModelRunner:
143
slot = block_number * self.block_size + block_offset
144
slot_mapping[-1].append(slot)
145
127
- max_prompt_len = max(prompt_lens)
146
+ max_prompt_len = max(subquery_lens)
147
input_tokens = _make_tensor_with_pad(input_tokens,
148
max_prompt_len,
149
pad=0,
@@ -137,16 +156,39 @@ class ModelRunner:
156
max_prompt_len,
157
pad=_PAD_SLOT_ID,
158
dtype=torch.long)
159
+ context_lens_tensor = torch.tensor(context_lens,
160
+ dtype=torch.int,
161
+ device='cuda')
162
+ # Prepare prefix block tables
163
+ max_prompt_block_table_len = max(len(t) for t in prefix_block_tables)
164
+ block_tables = _make_tensor_with_pad(
165
+ prefix_block_tables,
166
+ max_len=max_prompt_block_table_len,
167
+ pad=0,
168
+ dtype=torch.int,
169
+ )
170
+ start_loc_tensor = torch.arange(0,
171
+ len(prompt_lens) * max_prompt_len,
172
+ max_prompt_len,
173
+ dtype=torch.long,
174
+ device='cuda')
175
+ prompt_lens_tensor = torch.tensor(prompt_lens,
176
+ dtype=torch.long,
177
+ device='cuda')
178
179
input_metadata = InputMetadata(
180
is_prompt=True,
181
slot_mapping=slot_mapping,
182
+ prompt_lens=prompt_lens_tensor,
183
+ max_seq_len=max_prompt_len,
184
+ start_loc=start_loc_tensor,
185
max_context_len=None,
145
- context_lens=None,
146
- block_tables=None,
186
+ context_lens=context_lens_tensor,
187
+ block_tables=block_tables,
188
use_cuda_graph=False,
189
)
149
- return input_tokens, input_positions, input_metadata, prompt_lens
190
+ return (input_tokens, input_positions, input_metadata, prompt_lens,
191
+ subquery_lens)
192
193
def _prepare_decode(
194
self,
@@ -248,6 +290,9 @@ class ModelRunner:
290
input_metadata = InputMetadata(
291
is_prompt=False,
292
slot_mapping=slot_mapping,
293
+ prompt_lens=None,
294
+ max_seq_len=None,
295
+ start_loc=None,
296
max_context_len=max_context_len,
297
context_lens=context_lens,
298
block_tables=block_tables,
@@ -259,6 +304,7 @@ class ModelRunner:
304
self,
305
seq_group_metadata_list: List[SequenceGroupMetadata],
306
prompt_lens: List[int],
307
+ subquery_lens: Optional[List[int]],
308
) -> SamplingMetadata:
309
seq_groups: List[Tuple[List[int], SamplingParams]] = []
310
selected_token_indices: List[int] = []
@@ -266,7 +312,7 @@ class ModelRunner:
312
categorized_sample_indices = {t: [] for t in SamplingType}
313
categorized_sample_indices_start_idx = 0
314
269
- max_prompt_len = max(prompt_lens) if prompt_lens else 1
315
+ max_subquery_len = max(subquery_lens) if subquery_lens else 1
316
for i, seq_group_metadata in enumerate(seq_group_metadata_list):
317
seq_ids = list(seq_group_metadata.seq_data.keys())
318
sampling_params = seq_group_metadata.sampling_params
@@ -274,10 +320,11 @@ class ModelRunner:
320
321
if seq_group_metadata.is_prompt:
322
assert len(seq_ids) == 1
277
- prompt_len = prompt_lens[i]
323
+ assert subquery_lens is not None
324
+ subquery_len = subquery_lens[i]
325
if sampling_params.prompt_logprobs is not None:
326
# NOTE: prompt token positions do not need sample, skip
280
- categorized_sample_indices_start_idx += prompt_len - 1
327
+ categorized_sample_indices_start_idx += subquery_len - 1
328
329
categorized_sample_indices[
330
sampling_params.sampling_type].append(
@@ -287,10 +334,10 @@ class ModelRunner:
334
if sampling_params.prompt_logprobs is not None:
335
selected_token_indices.extend(
336
range(selected_token_start_idx,
290
- selected_token_start_idx + prompt_len - 1))
337
+ selected_token_start_idx + subquery_len - 1))
338
selected_token_indices.append(selected_token_start_idx +
292
- prompt_len - 1)
293
- selected_token_start_idx += max_prompt_len
339
+ subquery_len - 1)
340
+ selected_token_start_idx += max_subquery_len
341
else:
342
num_seqs = len(seq_ids)
343
selected_token_indices.extend(
@@ -335,14 +382,16 @@ class ModelRunner:
382
is_prompt = seq_group_metadata_list[0].is_prompt
383
# Prepare input tensors.
384
if is_prompt:
338
- (input_tokens, input_positions, input_metadata,
339
- prompt_lens) = self._prepare_prompt(seq_group_metadata_list)
385
+ (input_tokens, input_positions, input_metadata, prompt_lens,
386
+ subquery_lens) = self._prepare_prompt(seq_group_metadata_list)
387
else:
388
(input_tokens, input_positions, input_metadata
389
) = self._prepare_decode(seq_group_metadata_list)
390
+ subquery_lens = None
391
prompt_lens = []
392
sampling_metadata = self._prepare_sample(seq_group_metadata_list,
345
- prompt_lens)
393
+ prompt_lens,
394
+ subquery_lens)
395
396
def get_size_or_none(x: Optional[torch.Tensor]):
397
return x.size() if x is not None else None
@@ -359,6 +408,12 @@ class ModelRunner:
408
input_metadata.is_prompt,
409
"slot_mapping_size":
410
get_size_or_none(input_metadata.slot_mapping),
411
+ "prompt_lens_size":
412
+ get_size_or_none(input_metadata.prompt_lens),
413
+ "max_seq_len":
414
+ input_metadata.max_seq_len,
415
+ "start_loc_size":
416
+ get_size_or_none(input_metadata.start_loc),
417
"max_context_len":
418
input_metadata.max_context_len,
419
"context_lens_size":
@@ -376,6 +431,10 @@ class ModelRunner:
431
broadcast(input_positions, src=0)
432
if input_metadata.slot_mapping is not None:
433
broadcast(input_metadata.slot_mapping, src=0)
434
+ if input_metadata.prompt_lens is not None:
435
+ broadcast(input_metadata.prompt_lens, src=0)
436
+ if input_metadata.start_loc is not None:
437
+ broadcast(input_metadata.start_loc, src=0)
438
if input_metadata.context_lens is not None:
439
broadcast(input_metadata.context_lens, src=0)
440
if input_metadata.block_tables is not None:
@@ -400,6 +459,20 @@ class ModelRunner:
459
broadcast(slot_mapping, src=0)
460
else:
461
slot_mapping = None
462
+ if py_data["prompt_lens_size"] is not None:
463
+ prompt_lens = torch.empty(*py_data["prompt_lens_size"],
464
+ dtype=torch.long,
465
+ device="cuda")
466
+ broadcast(prompt_lens, src=0)
467
+ else:
468
+ prompt_lens = None
469
+ if py_data["start_loc_size"] is not None:
470
+ start_loc = torch.empty(*py_data["start_loc_size"],
471
+ dtype=torch.long,
472
+ device="cuda")
473
+ broadcast(start_loc, src=0)
474
+ else:
475
+ start_loc = None
476
if py_data["context_lens_size"] is not None:
477
context_lens = torch.empty(*py_data["context_lens_size"],
478
dtype=torch.int,
@@ -422,6 +495,9 @@ class ModelRunner:
495
input_metadata = InputMetadata(
496
is_prompt=py_data["is_prompt"],
497
slot_mapping=slot_mapping,
498
+ prompt_lens=prompt_lens,
499
+ max_seq_len=py_data["max_seq_len"],
500
+ start_loc=start_loc,
501
max_context_len=py_data["max_context_len"],
502
context_lens=context_lens,
503
block_tables=block_tables,
@@ -534,6 +610,9 @@ class ModelRunner:
610
input_metadata = InputMetadata(
611
is_prompt=False,
612
slot_mapping=slot_mapping[:batch_size],
613
+ prompt_lens=None,
614
+ max_seq_len=None,
615
+ start_loc=None,
616
max_context_len=self.max_context_len_to_capture,
617
context_lens=context_lens[:batch_size],
618
block_tables=block_tables[:batch_size],