@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d10f8e1d43
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[Experimental] Prefix Caching Support (#1669)

Co-authored-by: DouHappy <2278958187@qq.com> Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>

shiyi.c_98 committed Jan 17, 2024 at 16:32 UTC d10f8e1d43bfb0656b6848ad0c681ecbdec812d6
20 files changed +1356 -71
.buildkite/test-pipeline.yaml
+4
@@ -31,6 +31,10 @@ steps:
31 - pytest -v -s models --forked
32 soft_fail: true
33
34 +- label: Prefix Caching Test
35 + commands:
36 + - pytest -v -s prefix_caching
37 +
38 - label: Samplers Test
39 command: pytest -v -s samplers --forked
40
examples/offline_inference_with_prefix.py new
+51
@@ -0,0 +1,51 @@
1 +from vllm import LLM, SamplingParams
2 +
3 +prefix = (
4 + "You are an expert school principal, skilled in effectively managing "
5 + "faculty and staff. Draft 10-15 questions for a potential first grade "
6 + "Head Teacher for my K-12, all-girls', independent school that emphasizes "
7 + "community, joyful discovery, and life-long learning. The candidate is "
8 + "coming in for a first-round panel interview for a 8th grade Math "
9 + "teaching role. They have 5 years of previous teaching experience "
10 + "as an assistant teacher at a co-ed, public school with experience "
11 + "in middle school math teaching. Based on these information, fulfill "
12 + "the following paragraph: ")
13 +
14 +# Sample prompts.
15 +prompts = [
16 + "Hello, my name is",
17 + "The president of the United States is",
18 + "The capital of France is",
19 + "The future of AI is",
20 +]
21 +# Create a sampling params object.
22 +sampling_params = SamplingParams(temperature=0.0)
23 +
24 +# Create an LLM.
25 +llm = LLM(model="facebook/opt-125m")
26 +
27 +generating_prompts = [prefix + prompt for prompt in prompts]
28 +
29 +# Generate texts from the prompts. The output is a list of RequestOutput objects
30 +# that contain the prompt, generated text, and other information.
31 +outputs = llm.generate(generating_prompts, sampling_params)
32 +# Print the outputs.
33 +for output in outputs:
34 + prompt = output.prompt
35 + generated_text = output.outputs[0].text
36 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
37 +
38 +print("-" * 80)
39 +
40 +# -1 since the last token can change when concatenating prompts.
41 +prefix_pos = len(llm.llm_engine.tokenizer.encode(prefix)) - 1
42 +
43 +# Generate with prefix
44 +outputs = llm.generate(generating_prompts, sampling_params,
45 + prefix_pos=[prefix_pos] * len(generating_prompts))
46 +
47 +# Print the outputs. You should see the same outputs as before
48 +for output in outputs:
49 + prompt = output.prompt
50 + generated_text = output.outputs[0].text
51 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
tests/kernels/test_prefix_prefill.py new
+168
@@ -0,0 +1,168 @@
1 +import random
2 +import pytest
3 +import time
4 +
5 +import torch
6 +from vllm.model_executor.layers.triton_kernel.prefix_prefill import (
7 + context_attention_fwd)
8 +from xformers import ops as xops
9 +from xformers.ops.fmha.attn_bias import BlockDiagonalCausalFromBottomRightMask
10 +
11 +NUM_HEADS = [12]
12 +HEAD_SIZES = [128]
13 +DTYPES = [torch.float16]
14 +
15 +
16 +@pytest.mark.parametrize("num_heads", NUM_HEADS)
17 +@pytest.mark.parametrize("head_size", HEAD_SIZES)
18 +@pytest.mark.parametrize("dtype", DTYPES)
19 +@torch.inference_mode()
20 +def test_contexted_kv_attention(
21 + num_heads: int,
22 + head_size: int,
23 + dtype: torch.dtype,
24 +) -> None:
25 + random.seed(0)
26 + torch.manual_seed(0)
27 + MAX_SEQ_LEN = 1024
28 + MAX_CTX_LEN = 1024
29 + BS = 10
30 + cache_size = 640
31 + block_size = 32
32 + max_block_per_request = 64
33 + subquery_lens = [random.randint(16, MAX_SEQ_LEN) for _ in range(BS)]
34 + ctx_lens = [random.randint(16, MAX_CTX_LEN) for _ in range(BS)]
35 + seq_lens = [a + b for a, b in zip(subquery_lens, ctx_lens)]
36 +
37 + num_tokens = sum(subquery_lens)
38 + query = torch.empty(num_tokens,
39 + num_heads,
40 + head_size,
41 + dtype=dtype,
42 + device='cuda')
43 + query.uniform_(-1e-3, 1e-3)
44 + output = torch.empty(num_tokens,
45 + num_heads,
46 + head_size,
47 + dtype=dtype,
48 + device='cuda')
49 +
50 + kv = torch.empty(sum(seq_lens),
51 + 2,
52 + num_heads,
53 + head_size,
54 + dtype=dtype,
55 + device='cuda')
56 + kv.uniform_(-1e-3, 1e-3)
57 + key, value = kv.unbind(dim=1)
58 +
59 + k_cache = torch.zeros(cache_size,
60 + block_size,
61 + num_heads,
62 + head_size,
63 + dtype=dtype,
64 + device='cuda')
65 + v_cache = torch.zeros(cache_size,
66 + block_size,
67 + num_heads,
68 + head_size,
69 + dtype=dtype,
70 + device='cuda')
71 + k = torch.zeros(sum(subquery_lens),
72 + num_heads,
73 + head_size,
74 + dtype=dtype,
75 + device='cuda')
76 + v = torch.zeros(sum(subquery_lens),
77 + num_heads,
78 + head_size,
79 + dtype=dtype,
80 + device='cuda')
81 + values = torch.arange(0, cache_size, dtype=torch.long, device='cuda')
82 + values = values[torch.randperm(cache_size)]
83 + block_table = values[:BS * max_block_per_request].view(
84 + BS, max_block_per_request)
85 + b_seq_len = torch.tensor(seq_lens, dtype=torch.long, device='cuda')
86 + b_ctx_len = torch.tensor(ctx_lens, dtype=torch.long, device='cuda')
87 + b_start_loc = torch.cumsum(torch.tensor([0] + subquery_lens[:-1],
88 + dtype=torch.long,
89 + device='cuda'),
90 + dim=0)
91 + max_input_len = MAX_SEQ_LEN
92 + # copy kv to cache
93 + b_seq_start_loc = torch.cumsum(torch.tensor([0] + seq_lens[:-1],
94 + dtype=torch.long,
95 + device='cuda'),
96 + dim=0)
97 + for i in range(BS):
98 + for j in range(subquery_lens[i]):
99 + k[b_start_loc[i] + j].copy_(key[b_seq_start_loc[i] + b_ctx_len[i] +
100 + j])
101 + v[b_start_loc[i] + j].copy_(value[b_seq_start_loc[i] +
102 + b_ctx_len[i] + j])
103 + cur_ctx = 0
104 + block_id = 0
105 + while cur_ctx < b_ctx_len[i]:
106 + start_loc = b_seq_start_loc[i] + cur_ctx
107 + if cur_ctx + block_size > b_ctx_len[i]:
108 + end_loc = b_seq_start_loc[i] + b_ctx_len[i]
109 + else:
110 + end_loc = start_loc + block_size
111 + start_slot = block_table[i, block_id] * block_size
112 + end_slot = start_slot + end_loc - start_loc
113 + k_cache.view(-1, num_heads, head_size)[start_slot:end_slot].copy_(
114 + key[start_loc:end_loc])
115 + v_cache.view(-1, num_heads, head_size)[start_slot:end_slot].copy_(
116 + value[start_loc:end_loc])
117 + cur_ctx += block_size
118 + block_id += 1
119 + # transpose K_cache[num_blocks, block_size, num_kv_heads, head_size]
120 + # to K_cache[num_blocks, num_kv_heads, head_size/8, block_size, 8]
121 + k_cache = k_cache.view(-1, block_size, num_heads, head_size // 8,
122 + 8).permute(0, 2, 3, 1, 4).contiguous()
123 + # transpose V_cache[num_blocks, block_size, num_kv_heads, head_size]
124 + # to V_cache[num_blocks, num_kv_heads, head_size, block_size]
125 + v_cache = v_cache.view(-1, block_size, num_heads,
126 + head_size).permute(0, 2, 3, 1).contiguous()
127 +
128 + context_attention_fwd(query, k, v, output, k_cache, v_cache, block_table,
129 + b_start_loc, b_seq_len, b_ctx_len, max_input_len)
130 + torch.cuda.synchronize()
131 + start_time = time.time()
132 + context_attention_fwd(query, k, v, output, k_cache, v_cache, block_table,
133 + b_start_loc, b_seq_len, b_ctx_len, max_input_len)
134 + torch.cuda.synchronize()
135 + end_time = time.time()
136 + print(f"triton Time: {(end_time - start_time)*1000:.2f} ms")
137 +
138 + scale = float(1.0 / (head_size**0.5))
139 +
140 + attn_op = xops.fmha.cutlass.FwOp()
141 +
142 + attn_bias = BlockDiagonalCausalFromBottomRightMask.from_seqlens(
143 + subquery_lens, seq_lens)
144 + output_ref = xops.memory_efficient_attention_forward(
145 + query.unsqueeze(0),
146 + key.unsqueeze(0),
147 + value.unsqueeze(0),
148 + attn_bias=attn_bias,
149 + p=0.0,
150 + scale=scale,
151 + op=attn_op,
152 + )
153 + torch.cuda.synchronize()
154 + start_time = time.time()
155 + output_ref = xops.memory_efficient_attention_forward(
156 + query.unsqueeze(0),
157 + key.unsqueeze(0),
158 + value.unsqueeze(0),
159 + attn_bias=attn_bias,
160 + p=0.0,
161 + scale=scale,
162 + op=attn_op,
163 + )
164 + torch.cuda.synchronize()
165 + end_time = time.time()
166 + print(f"xformers Time: {(end_time - start_time)*1000:.2f} ms")
167 + output_ref = output_ref.squeeze(0)
168 + assert torch.allclose(output_ref, output, atol=1e-6, rtol=0)
tests/prefix_caching/test_prefix_caching.py new
+41
@@ -0,0 +1,41 @@
1 +"""Compare the with and without prefix caching.
2 +
3 +Run `pytest tests/prefix_caching/test_prefix_caching.py`.
4 +"""
5 +import pytest
6 +
7 +from vllm import LLM, SamplingParams
8 +
9 +prefix = (
10 + "You are an expert school principal, skilled in effectively managing "
11 + "faculty and staff. Draft 10-15 questions for a potential first grade "
12 + "Head Teacher for my K-12, all-girls', independent school that emphasizes "
13 + "community, joyful discovery, and life-long learning. The candidate is "
14 + "coming in for a first-round panel interview for a 8th grade Math "
15 + "teaching role. They have 5 years of previous teaching experience "
16 + "as an assistant teacher at a co-ed, public school with experience "
17 + "in middle school math teaching. Based on these information, fulfill "
18 + "the following paragraph: ")
19 +
20 +
21 +@pytest.mark.parametrize("model", ["facebook/opt-125m"])
22 +@pytest.mark.parametrize("max_tokens", [16])
23 +def test_prefix_caching(
24 + example_prompts,
25 + model: str,
26 + max_tokens: int,
27 +):
28 + llm = LLM(model=model)
29 + # -1 since the last token can change when concatenating prompts.
30 + prefix_pos = len(llm.llm_engine.tokenizer.encode(prefix)) - 1
31 + prompts = [prefix + prompt for prompt in example_prompts]
32 + sampling_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
33 + outputs_without_prefix = llm.generate(prompts, sampling_params)
34 + outputs_with_prefix = llm.generate(prompts,
35 + sampling_params,
36 + prefix_pos=[prefix_pos] * len(prompts))
37 + for output_without_prefix, output_with_prefix in zip(
38 + outputs_without_prefix, outputs_with_prefix):
39 + assert (output_without_prefix.outputs[0].token_ids ==
40 + output_with_prefix.outputs[0].token_ids)
41 + assert len(llm.llm_engine.scheduler.prefix_pool.prefixes) == 1
tests/samplers/test_sampler.py
+12 -6
@@ -66,7 +66,8 @@ def test_sampler_all_greedy(seed: int):
66 prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
67
68 sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
69 - prompt_lens)
69 + prompt_lens,
70 + subquery_lens=prompt_lens)
71 sampler_output = sampler(embedding=None,
72 hidden_states=input_tensor,
73 sampling_metadata=sampling_metadata)
@@ -105,7 +106,8 @@ def test_sampler_all_random(seed: int):
106 prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
107
108 sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
108 - prompt_lens)
109 + prompt_lens,
110 + subquery_lens=prompt_lens)
111 sampler_output = sampler(embedding=None,
112 hidden_states=input_tensor,
113 sampling_metadata=sampling_metadata)
@@ -140,7 +142,8 @@ def test_sampler_all_beam(seed: int):
142 prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
143
144 sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
143 - prompt_lens)
145 + prompt_lens,
146 + subquery_lens=prompt_lens)
147 sampler(embedding=None,
148 hidden_states=input_tensor,
149 sampling_metadata=sampling_metadata)
@@ -193,7 +196,8 @@ def test_sampler_mixed(seed: int):
196 prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
197
198 sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
196 - prompt_lens)
199 + prompt_lens,
200 + subquery_lens=prompt_lens)
201 sampler_output = sampler(embedding=None,
202 hidden_states=input_tensor,
203 sampling_metadata=sampling_metadata)
@@ -234,7 +238,8 @@ def test_sampler_logits_processors(seed: int):
238 prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
239
240 sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
237 - prompt_lens)
241 + prompt_lens,
242 + subquery_lens=prompt_lens)
243 sampler_output = sampler(embedding=None,
244 hidden_states=input_tensor,
245 sampling_metadata=sampling_metadata)
@@ -288,7 +293,8 @@ def test_sampler_top_k_top_p(seed: int):
293 prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
294
295 sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
291 - prompt_lens)
296 + prompt_lens,
297 + subquery_lens=prompt_lens)
298
299 sample_probs = None
300
tests/worker/test_model_runner.py
+3 -2
@@ -33,11 +33,12 @@ def test_prepare_prompt():
33 expected_selected_token_indices.append(selected_token_start_idx +
34 prompt_len - 1)
35 selected_token_start_idx += max_seq_len
36 - input_tokens, input_positions, _, return_prompt_lens = (
36 + input_tokens, input_positions, _, return_prompt_lens, _ = (
37 model_runner._prepare_prompt(seq_group_metadata_list))
38 assert return_prompt_lens == prompt_lens
39 sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
40 - prompt_lens)
40 + prompt_lens,
41 + subquery_lens=prompt_lens)
42 assert input_tokens.shape == (batch_size, max_seq_len)
43 assert input_positions.shape == (batch_size, max_seq_len)
44 torch.testing.assert_close(input_tokens, input_positions)
vllm/block.py
+4
@@ -66,3 +66,7 @@ class PhysicalTokenBlock:
66 return (f'PhysicalTokenBlock(device={self.device}, '
67 f'block_number={self.block_number}, '
68 f'ref_count={self.ref_count})')
69 +
70 +
71 +# Mapping: logical block number -> physical block.
72 +BlockTable = List[PhysicalTokenBlock]
vllm/core/block_manager.py
+42 -5
@@ -2,13 +2,10 @@
2 import enum
3 from typing import Dict, List, Optional, Set, Tuple
4
5 -from vllm.block import PhysicalTokenBlock
5 +from vllm.block import BlockTable, PhysicalTokenBlock
6 from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
7 from vllm.utils import Device
8
9 -# Mapping: logical block number -> physical block.
10 -BlockTable = List[PhysicalTokenBlock]
11 -
9
10 class BlockAllocator:
11 """Manages free physical token blocks for a device.
@@ -105,6 +102,10 @@ class BlockSpaceManager:
102 # the same prompt. This may not be true for preempted sequences.
103 seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
104 num_required_blocks = len(seq.logical_token_blocks)
105 +
106 + if seq_group.prefix is not None and seq_group.prefix.allocated:
107 + num_required_blocks -= seq_group.prefix.get_num_blocks()
108 +
109 if self.block_sliding_window is not None:
110 num_required_blocks = min(num_required_blocks,
111 self.block_sliding_window)
@@ -125,8 +126,21 @@ class BlockSpaceManager:
126 seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
127
128 # Allocate new physical token blocks that will store the prompt tokens.
129 + num_prompt_blocks = len(seq.logical_token_blocks)
130 +
131 block_table: BlockTable = []
129 - for logical_idx in range(len(seq.logical_token_blocks)):
132 + prefix_block_table: BlockTable = []
133 + num_prefix_blocks = 0
134 +
135 + prefix = seq_group.prefix
136 + if prefix is not None and prefix.allocated:
137 + # Prefix has already been allocated. Use the existing block table.
138 + num_prompt_blocks -= prefix.get_num_blocks()
139 + for block in prefix.block_table:
140 + block.ref_count += seq_group.num_seqs()
141 + block_table.append(block)
142 +
143 + for logical_idx in range(num_prompt_blocks):
144 if (self.block_sliding_window is not None
145 and logical_idx >= self.block_sliding_window):
146 block = block_table[logical_idx % self.block_sliding_window]
@@ -136,6 +150,15 @@ class BlockSpaceManager:
150 block.ref_count = seq_group.num_seqs()
151 block_table.append(block)
152
153 + if prefix is not None and not prefix.allocated:
154 + # Allocate blocks for the prefix, we will compute the prefix's
155 + # KV cache in this run.
156 + num_prefix_blocks = prefix.get_num_blocks()
157 + prefix_block_table = block_table[:num_prefix_blocks]
158 + for block in prefix_block_table:
159 + block.ref_count += 1
160 + prefix.set_block_table(prefix_block_table)
161 +
162 # Assign the block table for each sequence.
163 for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
164 self.block_tables[seq.seq_id] = block_table.copy()
@@ -210,10 +233,18 @@ class BlockSpaceManager:
233
234 def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
235 # CPU block -> GPU block.
236 + if seq_group.prefix is not None:
237 + # make sure to swap in the prefix first
238 + assert seq_group.prefix.allocated and seq_group.prefix.computed
239 +
240 mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
241 for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
242 new_block_table: BlockTable = []
243 block_table = self.block_tables[seq.seq_id]
244 + if seq_group.prefix is not None:
245 + for block in seq_group.prefix.block_table:
246 + new_block_table.append(block)
247 + block.ref_count += 1
248
249 for cpu_block in block_table:
250 if cpu_block in mapping:
@@ -245,6 +276,12 @@ class BlockSpaceManager:
276 block_table = self.block_tables[seq.seq_id]
277
278 for gpu_block in block_table:
279 + if (seq_group.prefix is not None
280 + and gpu_block in seq_group.prefix.block_table):
281 + # NOTE: We do not swap out the prefix blocks for now.
282 + self.gpu_allocator.free(gpu_block)
283 + continue
284 +
285 if gpu_block in mapping:
286 cpu_block = mapping[gpu_block]
287 cpu_block.ref_count += 1
vllm/core/scheduler.py
+5
@@ -9,6 +9,7 @@ from vllm.core.policy import PolicyFactory
9 from vllm.logger import init_logger
10 from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
11 SequenceGroupMetadata, SequenceStatus)
12 +from vllm.prefix import PrefixPool
13
14 logger = init_logger(__name__)
15
@@ -76,6 +77,9 @@ class Scheduler:
77 num_cpu_blocks=self.cache_config.num_cpu_blocks,
78 sliding_window=self.cache_config.sliding_window)
79
80 + # Create the prefix pool to cache the prefixes.
81 + self.prefix_pool = PrefixPool(self.cache_config.block_size)
82 +
83 # Sequence groups in the WAITING state.
84 self.waiting: Deque[SequenceGroup] = deque()
85 # Sequence groups in the RUNNING state.
@@ -316,6 +320,7 @@ class Scheduler:
320 seq_data=seq_data,
321 sampling_params=seq_group.sampling_params,
322 block_tables=block_tables,
323 + prefix=seq_group.prefix,
324 )
325 seq_group_metadata_list.append(seq_group_metadata)
326 return seq_group_metadata_list, scheduler_outputs
vllm/engine/async_llm_engine.py
+13 -3
@@ -371,6 +371,7 @@ class AsyncLLMEngine:
371 sampling_params: SamplingParams,
372 prompt_token_ids: Optional[List[int]] = None,
373 arrival_time: Optional[float] = None,
374 + prefix_pos: Optional[int] = None,
375 ) -> AsyncStream:
376 if self.log_requests:
377 shortened_prompt = prompt
@@ -383,6 +384,7 @@ class AsyncLLMEngine:
384 max_log_len]
385 logger.info(f"Received request {request_id}: "
386 f"prompt: {shortened_prompt!r}, "
387 + f"prefix_pos: {prefix_pos},"
388 f"sampling params: {sampling_params}, "
389 f"prompt token ids: {shortened_token_ids}.")
390
@@ -401,7 +403,8 @@ class AsyncLLMEngine:
403 prompt=prompt,
404 sampling_params=sampling_params,
405 prompt_token_ids=prompt_token_ids,
404 - arrival_time=arrival_time)
406 + arrival_time=arrival_time,
407 + prefix_pos=prefix_pos)
408
409 return stream
410
@@ -410,7 +413,8 @@ class AsyncLLMEngine:
413 prompt: Optional[str],
414 sampling_params: SamplingParams,
415 request_id: str,
413 - prompt_token_ids: Optional[List[int]] = None
416 + prompt_token_ids: Optional[List[int]] = None,
417 + prefix_pos: Optional[int] = None,
418 ) -> AsyncIterator[RequestOutput]:
419 """Generate outputs for a request.
420
@@ -425,6 +429,11 @@ class AsyncLLMEngine:
429 request_id: The unique id of the request.
430 prompt_token_ids: The token IDs of the prompt. If None, we
431 use the tokenizer to convert the prompts to token IDs.
432 + prefix_pos: If not None, we use the given position as the prefix
433 + position for each prompt. We will cache the prefix's KV
434 + cache and reuse it for the next request with the same prefix.
435 + This is an experimental feature, and may be replaced with
436 + automatic prefix caching in the future.
437
438 Yields:
439 The output `RequestOutput` objects from the LLMEngine for the
@@ -482,7 +491,8 @@ class AsyncLLMEngine:
491 prompt,
492 sampling_params,
493 prompt_token_ids=prompt_token_ids,
485 - arrival_time=arrival_time)
494 + arrival_time=arrival_time,
495 + prefix_pos=prefix_pos)
496
497 async for request_output in stream:
498 yield request_output
vllm/engine/llm_engine.py
+17 -1
@@ -337,6 +337,7 @@ class LLMEngine:
337 sampling_params: SamplingParams,
338 prompt_token_ids: Optional[List[int]] = None,
339 arrival_time: Optional[float] = None,
340 + prefix_pos: Optional[int] = None,
341 ) -> None:
342 """Add a request to the engine's request pool.
343
@@ -353,6 +354,11 @@ class LLMEngine:
354 use the tokenizer to convert the prompts to token IDs.
355 arrival_time: The arrival time of the request. If None, we use
356 the current monotonic time.
357 + prefix_pos: If not None, we use the given position as the prefix
358 + position for each prompt. We will cache the prefix's KV
359 + cache and reuse it for the next request with the same prefix.
360 + This is an experimental feature, and may be replaced with
361 + automatic prefix caching in the future.
362
363 Details:
364 - Set arrival_time to the current time if it is None.
@@ -389,9 +395,13 @@ class LLMEngine:
395 seq_id = next(self.seq_counter)
396 seq = Sequence(seq_id, prompt, prompt_token_ids, block_size)
397
398 + # Check whether the input specifies prefix
399 + prefix = self.scheduler.prefix_pool.add_or_get_prefix(
400 + prompt_token_ids[:prefix_pos]) if prefix_pos is not None else None
401 +
402 # Create the sequence group.
403 seq_group = SequenceGroup(request_id, [seq], sampling_params,
394 - arrival_time)
404 + arrival_time, prefix)
405
406 # Add the sequence group to the scheduler.
407 self.scheduler.add_seq_group(seq_group)
@@ -662,6 +672,12 @@ class LLMEngine:
672 request_output = RequestOutput.from_seq_group(seq_group)
673 request_outputs.append(request_output)
674
675 + # Update prefix state, now all the uncomputed prefixes are computed.
676 + for seq_group in scheduled_seq_groups:
677 + if (seq_group.prefix is not None and seq_group.prefix.allocated
678 + and not seq_group.prefix.computed):
679 + seq_group.prefix.computed = True
680 +
681 if self.log_stats:
682 # Log the system stats.
683 self._log_system_stats(scheduler_outputs.prompt_run,
vllm/entrypoints/api_server.py
+5 -1
@@ -33,11 +33,15 @@ async def generate(request: Request) -> Response:
33 """
34 request_dict = await request.json()
35 prompt = request_dict.pop("prompt")
36 + prefix_pos = request_dict.pop("prefix_pos", None)
37 stream = request_dict.pop("stream", False)
38 sampling_params = SamplingParams(**request_dict)
39 request_id = random_uuid()
40
40 - results_generator = engine.generate(prompt, sampling_params, request_id)
41 + results_generator = engine.generate(prompt,
42 + sampling_params,
43 + request_id,
44 + prefix_pos=prefix_pos)
45
46 # Streaming case
47 async def stream_results() -> AsyncGenerator[bytes, None]:
vllm/entrypoints/llm.py
+14 -3
@@ -120,6 +120,7 @@ class LLM:
120 prompts: Optional[Union[str, List[str]]] = None,
121 sampling_params: Optional[SamplingParams] = None,
122 prompt_token_ids: Optional[List[List[int]]] = None,
123 + prefix_pos: Optional[Union[int, List[int]]] = None,
124 use_tqdm: bool = True,
125 ) -> List[RequestOutput]:
126 """Generates the completions for the input prompts.
@@ -134,6 +135,11 @@ class LLM:
135 None, we use the default sampling parameters.
136 prompt_token_ids: A list of token IDs for the prompts. If None, we
137 use the tokenizer to convert the prompts to token IDs.
138 + prefix_pos: If not None, we use the given position as the prefix
139 + position for each prompt. We will cache the prefix's KV
140 + cache and reuse it for the next request with the same prefix.
141 + This is an experimental feature, and may be replaced with
142 + automatic prefix caching in the future.
143 use_tqdm: Whether to use tqdm to display the progress bar.
144
145 Returns:
@@ -159,9 +165,10 @@ class LLM:
165 prompt_token_ids)
166 for i in range(num_requests):
167 prompt = prompts[i] if prompts is not None else None
168 + prefix_pos_i = prefix_pos[i] if prefix_pos is not None else None
169 token_ids = None if prompt_token_ids is None else prompt_token_ids[
170 i]
164 - self._add_request(prompt, sampling_params, token_ids)
171 + self._add_request(prompt, sampling_params, token_ids, prefix_pos_i)
172 return self._run_engine(use_tqdm)
173
174 def _add_request(
@@ -169,10 +176,14 @@ class LLM:
176 prompt: Optional[str],
177 sampling_params: SamplingParams,
178 prompt_token_ids: Optional[List[int]],
179 + prefix_pos: Optional[int] = None,
180 ) -> None:
181 request_id = str(next(self.request_counter))
174 - self.llm_engine.add_request(request_id, prompt, sampling_params,
175 - prompt_token_ids)
182 + self.llm_engine.add_request(request_id,
183 + prompt,
184 + sampling_params,
185 + prompt_token_ids,
186 + prefix_pos=prefix_pos)
187
188 def _run_engine(self, use_tqdm: bool) -> List[RequestOutput]:
189 # Initialize tqdm.
vllm/model_executor/input_metadata.py
+6
@@ -18,12 +18,18 @@ class InputMetadata:
18 self,
19 is_prompt: bool,
20 slot_mapping: torch.Tensor,
21 + prompt_lens: Optional[torch.Tensor],
22 + max_seq_len: Optional[int],
23 + start_loc: Optional[torch.Tensor],
24 max_context_len: Optional[int],
25 context_lens: Optional[torch.Tensor],
26 block_tables: Optional[torch.Tensor],
27 use_cuda_graph: bool,
28 ) -> None:
29 self.is_prompt = is_prompt
30 + self.prompt_lens = prompt_lens
31 + self.max_seq_len = max_seq_len
32 + self.start_loc = start_loc
33 self.max_context_len = max_context_len
34 self.slot_mapping = slot_mapping
35 self.context_lens = context_lens
vllm/model_executor/layers/attention.py
+55 -33
@@ -10,6 +10,8 @@ from xformers.ops.fmha.attn_bias import (BlockDiagonalCausalMask,
10 from vllm._C import ops
11 from vllm._C import cache_ops
12 from vllm.model_executor.input_metadata import InputMetadata
13 +from vllm.model_executor.layers.triton_kernel.prefix_prefill import (
14 + context_attention_fwd)
15 from vllm.utils import is_hip
16
17 _SUPPORTED_HEAD_SIZES = [64, 80, 96, 112, 128, 256]
@@ -115,45 +117,65 @@ class PagedAttention(nn.Module):
117 self.num_kv_heads,
118 self.num_queries_per_kv,
119 value.shape[-1])
120 + # normal attention
121 + if (key_cache is None or value_cache is None
122 + or input_metadata.block_tables.numel() == 0):
123 + # Set attention bias if not provided. This typically happens at
124 + # the very attention layer of every iteration.
125 + # FIXME(woosuk): This is a hack.
126 + if input_metadata.attn_bias is None:
127 + if self.alibi_slopes is None:
128 + attn_bias = BlockDiagonalCausalMask.from_seqlens(
129 + [seq_len] * batch_size)
130 + if self.sliding_window is not None:
131 + attn_bias = attn_bias.make_local_attention(
132 + self.sliding_window)
133 + input_metadata.attn_bias = attn_bias
134 + else:
135 + input_metadata.attn_bias = _make_alibi_bias(
136 + self.alibi_slopes, self.num_kv_heads, batch_size,
137 + seq_len, query.dtype)
138
119 - # Set attention bias if not provided. This typically happens at the
120 - # very attention layer of every iteration.
121 - # FIXME(woosuk): This is a hack.
122 - if input_metadata.attn_bias is None:
139 + # TODO(woosuk): Too many view operations. Let's try to reduce
140 + # them in the future for code readability.
141 if self.alibi_slopes is None:
124 - attn_bias = BlockDiagonalCausalMask.from_seqlens(
125 - [seq_len] * batch_size)
126 - if self.sliding_window is not None:
127 - attn_bias = attn_bias.make_local_attention(
128 - self.sliding_window)
129 - input_metadata.attn_bias = attn_bias
142 + query = query.unsqueeze(0)
143 + key = key.unsqueeze(0)
144 + value = value.unsqueeze(0)
145 else:
131 - input_metadata.attn_bias = _make_alibi_bias(
132 - self.alibi_slopes, self.num_kv_heads, batch_size,
133 - seq_len, query.dtype)
146 + query = query.unflatten(0, (batch_size, seq_len))
147 + key = key.unflatten(0, (batch_size, seq_len))
148 + value = value.unflatten(0, (batch_size, seq_len))
149
135 - # TODO(woosuk): Too many view operations. Let's try to reduce them
136 - # in the future for code readability.
137 - if self.alibi_slopes is None:
138 - query = query.unsqueeze(0)
139 - key = key.unsqueeze(0)
140 - value = value.unsqueeze(0)
150 + out = xops.memory_efficient_attention_forward(
151 + query,
152 + key,
153 + value,
154 + attn_bias=input_metadata.attn_bias,
155 + p=0.0,
156 + scale=self.scale,
157 + op=xops.fmha.MemoryEfficientAttentionFlashAttentionOp[0] if
158 + (is_hip()) else None,
159 + )
160 + output = out.view_as(query)
161 else:
142 - query = query.unflatten(0, (batch_size, seq_len))
143 - key = key.unflatten(0, (batch_size, seq_len))
144 - value = value.unflatten(0, (batch_size, seq_len))
162 + # prefix-enabled attention
163 + output = torch.empty_like(query)
164 + context_attention_fwd(
165 + query,
166 + key,
167 + value,
168 + output,
169 + key_cache,
170 + value_cache,
171 + input_metadata.block_tables, # [BS, max_block_per_request]
172 + input_metadata.start_loc,
173 + input_metadata.prompt_lens,
174 + input_metadata.context_lens,
175 + input_metadata.max_seq_len,
176 + getattr(self, "alibi_slopes", None),
177 + )
178
146 - out = xops.memory_efficient_attention_forward(
147 - query,
148 - key,
149 - value,
150 - attn_bias=input_metadata.attn_bias,
151 - p=0.0,
152 - scale=self.scale,
153 - op=xops.fmha.MemoryEfficientAttentionFlashAttentionOp[0] if
154 - (is_hip()) else None,
155 - )
156 - output = out.view_as(query)
179 else:
180 # Decoding run.
181 output = _paged_attention(
vllm/model_executor/layers/triton_kernel/__init__.py
vllm/model_executor/layers/triton_kernel/prefix_prefill.py new
+728
@@ -0,0 +1,728 @@
1 +# The kernels in this file are adapted from LightLLM's context_attention_fwd:
2 +# https://github.com/ModelTC/lightllm/blob/main/lightllm/models/llama/triton_kernel/context_flashattention_nopad.py
3 +
4 +import torch
5 +import triton
6 +import triton.language as tl
7 +
8 +if triton.__version__ >= "2.1.0":
9 +
10 + @triton.jit
11 + def _fwd_kernel(
12 + Q,
13 + K,
14 + V,
15 + K_cache,
16 + V_cache,
17 + B_Loc,
18 + sm_scale,
19 + B_Start_Loc,
20 + B_Seqlen,
21 + B_Ctxlen,
22 + block_size,
23 + x,
24 + Out,
25 + stride_b_loc_b,
26 + stride_b_loc_s,
27 + stride_qbs,
28 + stride_qh,
29 + stride_qd,
30 + stride_kbs,
31 + stride_kh,
32 + stride_kd,
33 + stride_vbs,
34 + stride_vh,
35 + stride_vd,
36 + stride_obs,
37 + stride_oh,
38 + stride_od,
39 + stride_k_cache_bs,
40 + stride_k_cache_h,
41 + stride_k_cache_d,
42 + stride_k_cache_bl,
43 + stride_k_cache_x,
44 + stride_v_cache_bs,
45 + stride_v_cache_h,
46 + stride_v_cache_d,
47 + stride_v_cache_bl,
48 + BLOCK_M: tl.constexpr,
49 + BLOCK_DMODEL: tl.constexpr,
50 + BLOCK_N: tl.constexpr,
51 + ):
52 + cur_batch = tl.program_id(0)
53 + cur_head = tl.program_id(1)
54 + start_m = tl.program_id(2)
55 +
56 + cur_batch_ctx_len = tl.load(B_Ctxlen + cur_batch)
57 + cur_batch_seq_len = tl.load(B_Seqlen + cur_batch)
58 + cur_batch_in_all_start_index = tl.load(B_Start_Loc + cur_batch)
59 +
60 + block_start_loc = BLOCK_M * start_m
61 +
62 + # initialize offsets
63 + offs_n = tl.arange(0, BLOCK_N)
64 + offs_d = tl.arange(0, BLOCK_DMODEL)
65 + offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
66 + off_q = (
67 + (cur_batch_in_all_start_index + offs_m[:, None]) * stride_qbs +
68 + cur_head * stride_qh + offs_d[None, :] * stride_qd)
69 +
70 + q = tl.load(
71 + Q + off_q,
72 + mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len,
73 + other=0.0)
74 +
75 + # # initialize pointer to m and l
76 + m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf")
77 + l_i = tl.zeros([BLOCK_M], dtype=tl.float32)
78 + acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32)
79 +
80 + for start_n in range(0, cur_batch_ctx_len, BLOCK_N):
81 + start_n = tl.multiple_of(start_n, BLOCK_N)
82 + # -- compute qk ----
83 + bn = tl.load(B_Loc + cur_batch * stride_b_loc_b +
84 + ((start_n + offs_n) // block_size) * stride_b_loc_s,
85 + mask=(start_n + offs_n) < cur_batch_ctx_len,
86 + other=0)
87 + off_k = (bn[None, :] * stride_k_cache_bs +
88 + cur_head * stride_k_cache_h +
89 + (offs_d[:, None] // x) * stride_k_cache_d +
90 + ((start_n + offs_n[None, :]) % block_size) *
91 + stride_k_cache_bl +
92 + (offs_d[:, None] % x) * stride_k_cache_x)
93 + off_v = (
94 + bn[:, None] * stride_v_cache_bs + cur_head * stride_v_cache_h +
95 + offs_d[None, :] * stride_v_cache_d +
96 + (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl)
97 + k = tl.load(K_cache + off_k,
98 + mask=(start_n + offs_n[None, :]) < cur_batch_ctx_len,
99 + other=0.0)
100 +
101 + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
102 + qk += tl.dot(q, k)
103 + qk = tl.where((start_n + offs_n[None, :]) < cur_batch_ctx_len, qk,
104 + float("-inf"))
105 + qk *= sm_scale
106 +
107 + # -- compute m_ij, p, l_ij
108 + m_ij = tl.max(qk, 1)
109 + p = tl.exp(qk - m_ij[:, None])
110 + l_ij = tl.sum(p, 1)
111 + # -- update m_i and l_i
112 + m_i_new = tl.maximum(m_i, m_ij)
113 + alpha = tl.exp(m_i - m_i_new)
114 + beta = tl.exp(m_ij - m_i_new)
115 + l_i_new = alpha * l_i + beta * l_ij
116 + # -- update output accumulator --
117 + # scale p
118 + p_scale = beta / l_i_new
119 + p = p * p_scale[:, None]
120 + # scale acc
121 + acc_scale = l_i / l_i_new * alpha
122 + acc = acc * acc_scale[:, None]
123 + # update acc
124 + v = tl.load(V_cache + off_v,
125 + mask=(start_n + offs_n[:, None]) < cur_batch_ctx_len,
126 + other=0.0)
127 +
128 + p = p.to(v.dtype)
129 + acc += tl.dot(p, v)
130 + # # update m_i and l_i
131 + l_i = l_i_new
132 + m_i = m_i_new
133 +
134 + off_k = (offs_n[None, :] * stride_kbs + cur_head * stride_kh +
135 + offs_d[:, None] * stride_kd)
136 + off_v = (offs_n[:, None] * stride_vbs + cur_head * stride_vh +
137 + offs_d[None, :] * stride_vd)
138 + k_ptrs = K + off_k
139 + v_ptrs = V + off_v
140 +
141 + block_mask = tl.where(
142 + block_start_loc < cur_batch_seq_len - cur_batch_ctx_len, 1, 0)
143 +
144 + for start_n in range(0, block_mask * (start_m + 1) * BLOCK_M, BLOCK_N):
145 + start_n = tl.multiple_of(start_n, BLOCK_N)
146 + # -- compute qk ----
147 + k = tl.load(k_ptrs +
148 + (cur_batch_in_all_start_index + start_n) * stride_kbs,
149 + mask=(start_n + offs_n[None, :]) <
150 + cur_batch_seq_len - cur_batch_ctx_len,
151 + other=0.0)
152 +
153 + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
154 + qk += tl.dot(q, k)
155 + qk *= sm_scale
156 + qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk,
157 + float("-inf"))
158 +
159 + # -- compute m_ij, p, l_ij
160 + m_ij = tl.max(qk, 1)
161 + p = tl.exp(qk - m_ij[:, None])
162 + l_ij = tl.sum(p, 1)
163 + # -- update m_i and l_i
164 + m_i_new = tl.maximum(m_i, m_ij)
165 + alpha = tl.exp(m_i - m_i_new)
166 + beta = tl.exp(m_ij - m_i_new)
167 + l_i_new = alpha * l_i + beta * l_ij
168 + # -- update output accumulator --
169 + # scale p
170 + p_scale = beta / l_i_new
171 + p = p * p_scale[:, None]
172 + # scale acc
173 + acc_scale = l_i / l_i_new * alpha
174 + acc = acc * acc_scale[:, None]
175 + # update acc
176 + v = tl.load(v_ptrs +
177 + (cur_batch_in_all_start_index + start_n) * stride_vbs,
178 + mask=(start_n + offs_n[:, None]) <
179 + cur_batch_seq_len - cur_batch_ctx_len,
180 + other=0.0)
181 +
182 + p = p.to(v.dtype)
183 + acc += tl.dot(p, v)
184 + # update m_i and l_i
185 + l_i = l_i_new
186 + m_i = m_i_new
187 + # initialize pointers to output
188 + off_o = (
189 + (cur_batch_in_all_start_index + offs_m[:, None]) * stride_obs +
190 + cur_head * stride_oh + offs_d[None, :] * stride_od)
191 + out_ptrs = Out + off_o
192 + tl.store(out_ptrs,
193 + acc,
194 + mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len)
195 + return
196 +
197 + @triton.jit
198 + def _fwd_kernel_flash_attn_v2(
199 + Q,
200 + K,
201 + V,
202 + K_cache,
203 + V_cache,
204 + B_Loc,
205 + sm_scale,
206 + B_Start_Loc,
207 + B_Seqlen,
208 + B_Ctxlen,
209 + block_size,
210 + x,
211 + Out,
212 + stride_b_loc_b,
213 + stride_b_loc_s,
214 + stride_qbs,
215 + stride_qh,
216 + stride_qd,
217 + stride_kbs,
218 + stride_kh,
219 + stride_kd,
220 + stride_vbs,
221 + stride_vh,
222 + stride_vd,
223 + stride_obs,
224 + stride_oh,
225 + stride_od,
226 + stride_k_cache_bs,
227 + stride_k_cache_h,
228 + stride_k_cache_d,
229 + stride_k_cache_bl,
230 + stride_k_cache_x,
231 + stride_v_cache_bs,
232 + stride_v_cache_h,
233 + stride_v_cache_d,
234 + stride_v_cache_bl,
235 + BLOCK_M: tl.constexpr,
236 + BLOCK_DMODEL: tl.constexpr,
237 + BLOCK_N: tl.constexpr,
238 + ):
239 + cur_batch = tl.program_id(0)
240 + cur_head = tl.program_id(1)
241 + start_m = tl.program_id(2)
242 +
243 + cur_batch_ctx_len = tl.load(B_Ctxlen + cur_batch)
244 + cur_batch_seq_len = tl.load(B_Seqlen + cur_batch)
245 + cur_batch_in_all_start_index = tl.load(B_Start_Loc + cur_batch)
246 +
247 + block_start_loc = BLOCK_M * start_m
248 +
249 + # initialize offsets
250 + offs_n = tl.arange(0, BLOCK_N)
251 + offs_d = tl.arange(0, BLOCK_DMODEL)
252 + offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
253 + off_q = (
254 + (cur_batch_in_all_start_index + offs_m[:, None]) * stride_qbs +
255 + cur_head * stride_qh + offs_d[None, :] * stride_qd)
256 +
257 + q = tl.load(
258 + Q + off_q,
259 + mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len,
260 + other=0.0)
261 +
262 + # # initialize pointer to m and l
263 + m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf")
264 + l_i = tl.zeros([BLOCK_M], dtype=tl.float32)
265 + acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32)
266 +
267 + for start_n in range(0, cur_batch_ctx_len, BLOCK_N):
268 + start_n = tl.multiple_of(start_n, BLOCK_N)
269 + # -- compute qk ----
270 + bn = tl.load(B_Loc + cur_batch * stride_b_loc_b +
271 + ((start_n + offs_n) // block_size) * stride_b_loc_s,
272 + mask=(start_n + offs_n) < cur_batch_ctx_len,
273 + other=0)
274 + off_k = (bn[None, :] * stride_k_cache_bs +
275 + cur_head * stride_k_cache_h +
276 + (offs_d[:, None] // x) * stride_k_cache_d +
277 + ((start_n + offs_n[None, :]) % block_size) *
278 + stride_k_cache_bl +
279 + (offs_d[:, None] % x) * stride_k_cache_x)
280 + off_v = (
281 + bn[:, None] * stride_v_cache_bs + cur_head * stride_v_cache_h +
282 + offs_d[None, :] * stride_v_cache_d +
283 + (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl)
284 + k = tl.load(K_cache + off_k,
285 + mask=(start_n + offs_n[None, :]) < cur_batch_ctx_len,
286 + other=0.0)
287 +
288 + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
289 + qk += tl.dot(q, k)
290 + qk = tl.where((start_n + offs_n[None, :]) < cur_batch_ctx_len, qk,
291 + float("-inf"))
292 + qk *= sm_scale
293 +
294 + # -- compute m_ij, p, l_ij
295 + m_ij = tl.max(qk, 1)
296 + m_i_new = tl.maximum(m_i, m_ij)
297 + p = tl.math.exp(qk - m_i_new[:, None])
298 + l_ij = tl.sum(p, 1)
299 + # -- update m_i and l_i
300 +
301 + alpha = tl.math.exp(m_i - m_i_new)
302 + l_i_new = alpha * l_i + l_ij
303 + # -- update output accumulator --
304 + # scale p
305 + # scale acc
306 + acc_scale = alpha
307 + # acc_scale = l_i / l_i_new * alpha
308 + acc = acc * acc_scale[:, None]
309 + # update acc
310 + v = tl.load(V_cache + off_v,
311 + mask=(start_n + offs_n[:, None]) < cur_batch_ctx_len,
312 + other=0.0)
313 +
314 + p = p.to(v.dtype)
315 + acc += tl.dot(p, v)
316 + # update m_i and l_i
317 + l_i = l_i_new
318 + m_i = m_i_new
319 +
320 + off_k = (offs_n[None, :] * stride_kbs + cur_head * stride_kh +
321 + offs_d[:, None] * stride_kd)
322 + off_v = (offs_n[:, None] * stride_vbs + cur_head * stride_vh +
323 + offs_d[None, :] * stride_vd)
324 + k_ptrs = K + off_k
325 + v_ptrs = V + off_v
326 +
327 + block_mask = tl.where(
328 + block_start_loc < cur_batch_seq_len - cur_batch_ctx_len, 1, 0)
329 +
330 + for start_n in range(0, block_mask * (start_m + 1) * BLOCK_M, BLOCK_N):
331 + start_n = tl.multiple_of(start_n, BLOCK_N)
332 + # -- compute qk ----
333 + k = tl.load(k_ptrs +
334 + (cur_batch_in_all_start_index + start_n) * stride_kbs,
335 + mask=(start_n + offs_n[None, :]) <
336 + cur_batch_seq_len - cur_batch_ctx_len,
337 + other=0.0)
338 +
339 + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
340 + qk += tl.dot(q, k)
341 + qk *= sm_scale
342 + qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk,
343 + float("-inf"))
344 +
345 + # -- compute m_ij, p, l_ij
346 + m_ij = tl.max(qk, 1)
347 + m_i_new = tl.maximum(m_i, m_ij)
348 + p = tl.math.exp(qk - m_i_new[:, None])
349 + l_ij = tl.sum(p, 1)
350 + # -- update m_i and l_i
351 +
352 + alpha = tl.math.exp(m_i - m_i_new)
353 + l_i_new = alpha * l_i + l_ij
354 + # -- update output accumulator --
355 + # scale p
356 + # scale acc
357 + acc_scale = alpha
358 + # acc_scale = l_i / l_i_new * alpha
359 + acc = acc * acc_scale[:, None]
360 + # update acc
361 + v = tl.load(v_ptrs +
362 + (cur_batch_in_all_start_index + start_n) * stride_vbs,
363 + mask=(start_n + offs_n[:, None]) <
364 + cur_batch_seq_len - cur_batch_ctx_len,
365 + other=0.0)
366 +
367 + p = p.to(v.dtype)
368 + acc += tl.dot(p, v)
369 + # update m_i and l_i
370 + l_i = l_i_new
371 + m_i = m_i_new
372 +
373 + # acc /= l_i[:, None]
374 + # initialize pointers to output
375 + off_o = (
376 + (cur_batch_in_all_start_index + offs_m[:, None]) * stride_obs +
377 + cur_head * stride_oh + offs_d[None, :] * stride_od)
378 + out_ptrs = Out + off_o
379 + tl.store(out_ptrs,
380 + acc,
381 + mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len)
382 + return
383 +
384 + @triton.jit
385 + def _fwd_kernel_alibi(
386 + Q,
387 + K,
388 + V,
389 + K_cache,
390 + V_cache,
391 + B_Loc,
392 + sm_scale,
393 + B_Start_Loc,
394 + B_Seqlen,
395 + B_Ctxlen,
396 + Alibi_slopes,
397 + block_size,
398 + x,
399 + Out,
400 + stride_b_loc_b,
401 + stride_b_loc_s,
402 + stride_qbs,
403 + stride_qh,
404 + stride_qd,
405 + stride_kbs,
406 + stride_kh,
407 + stride_kd,
408 + stride_vbs,
409 + stride_vh,
410 + stride_vd,
411 + stride_obs,
412 + stride_oh,
413 + stride_od,
414 + stride_k_cache_bs,
415 + stride_k_cache_h,
416 + stride_k_cache_d,
417 + stride_k_cache_bl,
418 + stride_k_cache_x,
419 + stride_v_cache_bs,
420 + stride_v_cache_h,
421 + stride_v_cache_d,
422 + stride_v_cache_bl,
423 + BLOCK_M: tl.constexpr,
424 + BLOCK_DMODEL: tl.constexpr,
425 + BLOCK_N: tl.constexpr,
426 + ):
427 + # attn_bias[]
428 + cur_batch = tl.program_id(0)
429 + cur_head = tl.program_id(1)
430 + start_m = tl.program_id(2)
431 +
432 + # cur_batch_seq_len: the length of prompts
433 + # cur_batch_ctx_len: the length of prefix
434 + # cur_batch_in_all_start_index: the start id of the dim=0
435 + cur_batch_ctx_len = tl.load(B_Ctxlen + cur_batch)
436 + cur_batch_seq_len = tl.load(B_Seqlen + cur_batch)
437 + cur_batch_in_all_start_index = tl.load(B_Start_Loc + cur_batch)
438 +
439 + block_start_loc = BLOCK_M * start_m
440 +
441 + # initialize offsets
442 + offs_n = tl.arange(0, BLOCK_N)
443 + offs_d = tl.arange(0, BLOCK_DMODEL)
444 + offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
445 + off_q = (
446 + (cur_batch_in_all_start_index + offs_m[:, None]) * stride_qbs +
447 + cur_head * stride_qh + offs_d[None, :] * stride_qd)
448 +
449 + q = tl.load(
450 + Q + off_q,
451 + mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len,
452 + other=0.0)
453 +
454 + # # initialize pointer to m and l
455 + m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf")
456 + l_i = tl.zeros([BLOCK_M], dtype=tl.float32)
457 + acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32)
458 +
459 + alibi_slope = tl.load(Alibi_slopes + cur_head)
460 + alibi_start_q = tl.arange(
461 + 0, BLOCK_M) + block_start_loc + cur_batch_ctx_len
462 + alibi_start_k = 0
463 + for start_n in range(0, cur_batch_ctx_len, BLOCK_N):
464 + start_n = tl.multiple_of(start_n, BLOCK_N)
465 + # -- compute qk ----
466 + bn = tl.load(B_Loc + cur_batch * stride_b_loc_b +
467 + ((start_n + offs_n) // block_size) * stride_b_loc_s,
468 + mask=(start_n + offs_n) < cur_batch_ctx_len,
469 + other=0)
470 + off_k = (bn[None, :] * stride_k_cache_bs +
471 + cur_head * stride_k_cache_h +
472 + (offs_d[:, None] // x) * stride_k_cache_d +
473 + ((start_n + offs_n[None, :]) % block_size) *
474 + stride_k_cache_bl +
475 + (offs_d[:, None] % x) * stride_k_cache_x)
476 + off_v = (
477 + bn[:, None] * stride_v_cache_bs + cur_head * stride_v_cache_h +
478 + offs_d[None, :] * stride_v_cache_d +
479 + (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl)
480 + k = tl.load(K_cache + off_k,
481 + mask=(start_n + offs_n[None, :]) < cur_batch_ctx_len,
482 + other=0.0)
483 +
484 + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
485 + qk += tl.dot(q, k)
486 + qk = tl.where((start_n + offs_n[None, :]) < cur_batch_ctx_len, qk,
487 + float("-inf"))
488 + qk *= sm_scale
489 +
490 + # load alibi
491 + alibi = (tl.arange(0, BLOCK_N)[None, :] + alibi_start_k -
492 + alibi_start_q[:, None]) * alibi_slope
493 + alibi = tl.where(
494 + (alibi <= 0) & (alibi_start_q[:, None] < cur_batch_seq_len),
495 + alibi, float("-inf"))
496 + qk += alibi
497 + alibi_start_k += BLOCK_N
498 +
499 + # -- compute m_ij, p, l_ij
500 + m_ij = tl.max(qk, 1)
501 + m_i_new = tl.maximum(m_i, m_ij)
502 + p = tl.math.exp(qk - m_i_new[:, None])
503 + l_ij = tl.sum(p, 1)
504 + # -- update m_i and l_i
505 +
506 + alpha = tl.math.exp(m_i - m_i_new)
507 + l_i_new = alpha * l_i + l_ij
508 + # -- update output accumulator --
509 + # scale p
510 + # scale acc
511 + acc_scale = alpha
512 + # acc_scale = l_i / l_i_new * alpha
513 + acc = acc * acc_scale[:, None]
514 + # update acc
515 + v = tl.load(V_cache + off_v,
516 + mask=(start_n + offs_n[:, None]) < cur_batch_ctx_len,
517 + other=0.0)
518 +
519 + p = p.to(v.dtype)
520 + acc += tl.dot(p, v, allow_tf32=False)
521 + # update m_i and l_i
522 + l_i = l_i_new
523 + m_i = m_i_new
524 +
525 + off_k = (offs_n[None, :] * stride_kbs + cur_head * stride_kh +
526 + offs_d[:, None] * stride_kd)
527 + off_v = (offs_n[:, None] * stride_vbs + cur_head * stride_vh +
528 + offs_d[None, :] * stride_vd)
529 + k_ptrs = K + off_k
530 + v_ptrs = V + off_v
531 +
532 + block_mask = tl.where(
533 + block_start_loc < cur_batch_seq_len - cur_batch_ctx_len, 1, 0)
534 +
535 + # init alibi
536 + alibi_slope = tl.load(Alibi_slopes + cur_head)
537 + alibi_start_q = tl.arange(
538 + 0, BLOCK_M) + block_start_loc + cur_batch_ctx_len
539 + alibi_start_k = cur_batch_ctx_len
540 + # # init debuger
541 + # offset_db_q = tl.arange(0, BLOCK_M) + block_start_loc
542 + # offset_db_k = tl.arange(0, BLOCK_N)
543 + # calc q[BLOCK_M, BLOCK_MODEL] mul k[prefix_len: , BLOCK_DMODEL]
544 + for start_n in range(0, block_mask * (start_m + 1) * BLOCK_M, BLOCK_N):
545 + start_n = tl.multiple_of(start_n, BLOCK_N)
546 + # -- compute qk ----
547 + k = tl.load(k_ptrs +
548 + (cur_batch_in_all_start_index + start_n) * stride_kbs,
549 + mask=(start_n + offs_n[None, :]) <
550 + cur_batch_seq_len - cur_batch_ctx_len,
551 + other=0.0)
552 +
553 + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
554 + qk += tl.dot(q, k, allow_tf32=False)
555 + qk *= sm_scale
556 + qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk,
557 + float("-inf"))
558 +
559 + # load alibi
560 + alibi = (tl.arange(0, BLOCK_N)[None, :] + alibi_start_k -
561 + alibi_start_q[:, None]) * alibi_slope
562 + alibi = tl.where(
563 + (alibi <= 0) & (alibi_start_q[:, None] < cur_batch_seq_len),
564 + alibi, float("-inf"))
565 + qk += alibi
566 + alibi_start_k += BLOCK_N
567 +
568 + # -- compute m_ij, p, l_ij
569 + m_ij = tl.max(qk, 1)
570 + m_i_new = tl.maximum(m_i, m_ij)
571 + p = tl.math.exp(qk - m_i_new[:, None])
572 + l_ij = tl.sum(p, 1)
573 + # -- update m_i and l_i
574 +
575 + alpha = tl.math.exp(m_i - m_i_new)
576 + l_i_new = alpha * l_i + l_ij
577 + # -- update output accumulator --
578 + # scale p
579 + # scale acc
580 + acc_scale = alpha
581 + # acc_scale = l_i / l_i_new * alpha
582 + acc = acc * acc_scale[:, None]
583 + # update acc
584 + v = tl.load(v_ptrs +
585 + (cur_batch_in_all_start_index + start_n) * stride_vbs,
586 + mask=(start_n + offs_n[:, None]) <
587 + cur_batch_seq_len - cur_batch_ctx_len,
588 + other=0.0)
589 +
590 + p = p.to(v.dtype)
591 + acc += tl.dot(p, v, allow_tf32=False)
592 + # update m_i and l_i
593 + l_i = l_i_new
594 + m_i = m_i_new
595 +
596 + acc = acc / l_i[:, None]
597 +
598 + # initialize pointers to output
599 + off_o = (
600 + (cur_batch_in_all_start_index + offs_m[:, None]) * stride_obs +
601 + cur_head * stride_oh + offs_d[None, :] * stride_od)
602 + out_ptrs = Out + off_o
603 + tl.store(out_ptrs,
604 + acc,
605 + mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len)
606 + return
607 +
608 + @torch.inference_mode()
609 + def context_attention_fwd(q,
610 + k,
611 + v,
612 + o,
613 + k_cache,
614 + v_cache,
615 + b_loc,
616 + b_start_loc,
617 + b_seq_len,
618 + b_ctx_len,
619 + max_input_len,
620 + alibi_slopes=None):
621 + BLOCK = 128
622 + # shape constraints
623 + Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1]
624 + assert Lq == Lk and Lk == Lv
625 + assert Lk in {16, 32, 64, 128}
626 +
627 + sm_scale = 1.0 / (Lq**0.5)
628 + batch, head = b_seq_len.shape[0], q.shape[1]
629 +
630 + grid = (batch, head, triton.cdiv(max_input_len, BLOCK)) # batch, head,
631 +
632 + num_warps = 8 if Lk <= 64 else 8
633 + if alibi_slopes is not None:
634 + _fwd_kernel_alibi[grid](
635 + q,
636 + k,
637 + v,
638 + k_cache,
639 + v_cache,
640 + b_loc,
641 + sm_scale,
642 + b_start_loc,
643 + b_seq_len,
644 + b_ctx_len,
645 + alibi_slopes,
646 + v_cache.shape[3],
647 + 8,
648 + o,
649 + b_loc.stride(0),
650 + b_loc.stride(1),
651 + q.stride(0),
652 + q.stride(1),
653 + q.stride(2),
654 + k.stride(0),
655 + k.stride(1),
656 + k.stride(2),
657 + v.stride(0),
658 + v.stride(1),
659 + v.stride(2),
660 + o.stride(0),
661 + o.stride(1),
662 + o.stride(2),
663 + k_cache.stride(0),
664 + k_cache.stride(1),
665 + k_cache.stride(2),
666 + k_cache.stride(3),
667 + k_cache.stride(
668 + 4
669 + ), #[num_blocks, num_kv_heads, head_size/x, block_size, x]
670 + v_cache.stride(0),
671 + v_cache.stride(1),
672 + v_cache.stride(2),
673 + v_cache.stride(
674 + 3), #[num_blocks, num_kv_heads, head_size, block_size]
675 + BLOCK_M=BLOCK,
676 + BLOCK_DMODEL=Lk,
677 + BLOCK_N=BLOCK,
678 + num_warps=num_warps,
679 + num_stages=1,
680 + )
681 + return
682 +
683 + _fwd_kernel[grid](
684 + q,
685 + k,
686 + v,
687 + k_cache,
688 + v_cache,
689 + b_loc,
690 + sm_scale,
691 + b_start_loc,
692 + b_seq_len,
693 + b_ctx_len,
694 + v_cache.shape[3],
695 + 8,
696 + o,
697 + b_loc.stride(0),
698 + b_loc.stride(1),
699 + q.stride(0),
700 + q.stride(1),
701 + q.stride(2),
702 + k.stride(0),
703 + k.stride(1),
704 + k.stride(2),
705 + v.stride(0),
706 + v.stride(1),
707 + v.stride(2),
708 + o.stride(0),
709 + o.stride(1),
710 + o.stride(2),
711 + k_cache.stride(0),
712 + k_cache.stride(1),
713 + k_cache.stride(2),
714 + k_cache.stride(3),
715 + k_cache.stride(
716 + 4), #[num_blocks, num_kv_heads, head_size/x, block_size, x]
717 + v_cache.stride(0),
718 + v_cache.stride(1),
719 + v_cache.stride(2),
720 + v_cache.stride(
721 + 3), #[num_blocks, num_kv_heads, head_size, block_size]
722 + BLOCK_M=BLOCK,
723 + BLOCK_DMODEL=Lk,
724 + BLOCK_N=BLOCK,
725 + num_warps=num_warps,
726 + num_stages=1,
727 + )
728 + return
vllm/prefix.py new
+87
@@ -0,0 +1,87 @@
1 +from typing import Dict, List, Sequence, Tuple, Optional
2 +
3 +from vllm.block import BlockTable
4 +
5 +
6 +class Prefix:
7 + """Data and states associated with a prefix of prompt tokens for multiple
8 + sequence groups.
9 +
10 + NOTE: This feature is experimental and may be replaced with automatic
11 + prefix caching in the future.
12 +
13 + Args:
14 + prefix_id: The id of the prefix in the prefix pool.
15 + token_ids: The token ids of the prefix.
16 + block_size: The block size of the executed model.
17 + """
18 +
19 + def __init__(
20 + self,
21 + token_ids: Sequence[int],
22 + block_size: int,
23 + ) -> None:
24 + self.token_ids = tuple(token_ids)
25 + self.block_size = block_size
26 + self.length = len(token_ids)
27 + self.hash = hash(token_ids)
28 + assert self.length % block_size == 0
29 + self.block_table: Optional[BlockTable] = None
30 + self.computed = False
31 +
32 + @property
33 + def allocated(self) -> bool:
34 + return self.block_table is not None
35 +
36 + def get_num_blocks(self) -> int:
37 + return self.length // self.block_size
38 +
39 + def get_block_numbers(self) -> List[int]:
40 + return [block.block_number for block in self.block_table]
41 +
42 + def get_length(self) -> int:
43 + return self.length
44 +
45 + def __hash__(self) -> int:
46 + return self.hash
47 +
48 + def set_block_table(self, block_table: BlockTable) -> None:
49 + self.block_table = block_table.copy()
50 +
51 +
52 +class PrefixPool:
53 + """Manages all the prompt prefixes.
54 +
55 + NOTE: This feature is experimental and may be replaced with automatic
56 + prefix caching in the future.
57 +
58 + Args:
59 + block_size: The block size of the executed model.
60 +
61 + Attributes:
62 + prefixes: A list of all the prefixes.
63 + block_size: The block size of the executed model.
64 + """
65 +
66 + def __init__(
67 + self,
68 + block_size: int,
69 + ) -> None:
70 + # TODO(zhuohan): Add a capacity limit to the prefix pool.
71 + self.prefixes: Dict[int, Prefix] = {}
72 + self.block_size = block_size
73 +
74 + def _truncate_token_ids(self, token_ids: Sequence[int]) -> Tuple[int]:
75 + new_length = len(token_ids) // self.block_size * self.block_size
76 + return tuple(token_ids[:new_length])
77 +
78 + def add_or_get_prefix(self, token_ids: Sequence[int]) -> Optional[Prefix]:
79 + token_ids = self._truncate_token_ids(token_ids)
80 + if len(token_ids) == 0:
81 + # Prefix is empty.
82 + return None
83 + prefix = Prefix(token_ids, self.block_size)
84 + prefix_hash = hash(prefix)
85 + if prefix_hash not in self.prefixes:
86 + self.prefixes[prefix_hash] = prefix
87 + return self.prefixes[prefix_hash]
vllm/sequence.py
+6 -1
@@ -4,6 +4,7 @@ import enum
4 from typing import Dict, List, Optional, Union
5
6 from vllm.block import LogicalTokenBlock
7 +from vllm.prefix import Prefix
8 from vllm.sampling_params import SamplingParams
9
10 PromptLogprobs = List[Optional[Dict[int, float]]]
@@ -236,11 +237,13 @@ class SequenceGroup:
237 seqs: List[Sequence],
238 sampling_params: SamplingParams,
239 arrival_time: float,
240 + prefix: Optional[Prefix] = None,
241 ) -> None:
242 self.request_id = request_id
243 self.seqs_dict = {seq.seq_id: seq for seq in seqs}
244 self.sampling_params = sampling_params
245 self.arrival_time = arrival_time
246 + self.prefix: Optional[Prefix] = prefix
247 self.prompt_logprobs: Optional[PromptLogprobs] = None
248
249 @property
@@ -327,7 +330,6 @@ class SequenceGroup:
330 class SequenceGroupMetadata:
331 """Metadata for a sequence group. Used to create `InputMetadata`.
332
330 -
333 Args:
334 request_id: The ID of the request.
335 is_prompt: Whether the request is at prompt stage.
@@ -335,6 +337,7 @@ class SequenceGroupMetadata:
337 sampling_params: The sampling parameters used to generate the outputs.
338 block_tables: The block tables. (Seq id -> list of physical block
339 numbers)
340 + prefix: The prefix of the prompt of the sequence group.
341 """
342
343 def __init__(
@@ -344,12 +347,14 @@ class SequenceGroupMetadata:
347 seq_data: Dict[int, SequenceData],
348 sampling_params: SamplingParams,
349 block_tables: Dict[int, List[int]],
350 + prefix: Optional[Prefix] = None,
351 ) -> None:
352 self.request_id = request_id
353 self.is_prompt = is_prompt
354 self.seq_data = seq_data
355 self.sampling_params = sampling_params
356 self.block_tables = block_tables
357 + self.prefix = prefix
358
359
360 class SequenceOutput:
vllm/worker/model_runner.py
+95 -16
@@ -74,13 +74,17 @@ class ModelRunner:
74 def _prepare_prompt(
75 self,
76 seq_group_metadata_list: List[SequenceGroupMetadata],
77 - ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, List[int]]:
77 + ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, List[int],
78 + List[int]]:
79 assert len(seq_group_metadata_list) > 0
80 input_tokens: List[List[int]] = []
81 input_positions: List[List[int]] = []
82 slot_mapping: List[List[int]] = []
83
84 prompt_lens: List[int] = []
85 + context_lens: List[int] = []
86 + subquery_lens: List[int] = []
87 + prefix_block_tables: List[List[int]] = []
88 for seq_group_metadata in seq_group_metadata_list:
89 assert seq_group_metadata.is_prompt
90 seq_ids = list(seq_group_metadata.seq_data.keys())
@@ -91,11 +95,23 @@ class ModelRunner:
95 prompt_tokens = seq_data.get_token_ids()
96 prompt_len = len(prompt_tokens)
97 prompt_lens.append(prompt_len)
98 + prefix_len = 0
99 + prefix = seq_group_metadata.prefix
100 + if prefix is not None and prefix.computed:
101 + prefix_len = prefix.get_length()
102 + prompt_tokens = prompt_tokens[prefix_len:]
103 + prefix_block_tables.append(prefix.get_block_numbers())
104 + else:
105 + prefix_block_tables.append([])
106 + # actual prompt lens
107 + context_lens.append(prefix_len)
108 + subquery_lens.append(prompt_len - prefix_len)
109
110 input_tokens.append(prompt_tokens)
111 # NOTE(woosuk): Here we assume that the first token in the prompt
112 # is always the first token in the sequence.
98 - input_positions.append(list(range(prompt_len)))
113 + input_positions.append(
114 + list(range(prefix_len, prefix_len + len(prompt_tokens))))
115
116 if seq_group_metadata.block_tables is None:
117 # During memory profiling, the block tables are not initialized
@@ -113,8 +129,11 @@ class ModelRunner:
129 # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
130 start_idx = 0
131 if self.sliding_window is not None:
132 + assert prefix_len == 0, (
133 + "Prefix caching is currently not supported with "
134 + "sliding window attention")
135 start_idx = max(0, prompt_len - self.sliding_window)
117 - for i in range(prompt_len):
136 + for i in range(prefix_len, prompt_len):
137 if i < start_idx:
138 slot_mapping[-1].append(_PAD_SLOT_ID)
139 continue
@@ -124,7 +143,7 @@ class ModelRunner:
143 slot = block_number * self.block_size + block_offset
144 slot_mapping[-1].append(slot)
145
127 - max_prompt_len = max(prompt_lens)
146 + max_prompt_len = max(subquery_lens)
147 input_tokens = _make_tensor_with_pad(input_tokens,
148 max_prompt_len,
149 pad=0,
@@ -137,16 +156,39 @@ class ModelRunner:
156 max_prompt_len,
157 pad=_PAD_SLOT_ID,
158 dtype=torch.long)
159 + context_lens_tensor = torch.tensor(context_lens,
160 + dtype=torch.int,
161 + device='cuda')
162 + # Prepare prefix block tables
163 + max_prompt_block_table_len = max(len(t) for t in prefix_block_tables)
164 + block_tables = _make_tensor_with_pad(
165 + prefix_block_tables,
166 + max_len=max_prompt_block_table_len,
167 + pad=0,
168 + dtype=torch.int,
169 + )
170 + start_loc_tensor = torch.arange(0,
171 + len(prompt_lens) * max_prompt_len,
172 + max_prompt_len,
173 + dtype=torch.long,
174 + device='cuda')
175 + prompt_lens_tensor = torch.tensor(prompt_lens,
176 + dtype=torch.long,
177 + device='cuda')
178
179 input_metadata = InputMetadata(
180 is_prompt=True,
181 slot_mapping=slot_mapping,
182 + prompt_lens=prompt_lens_tensor,
183 + max_seq_len=max_prompt_len,
184 + start_loc=start_loc_tensor,
185 max_context_len=None,
145 - context_lens=None,
146 - block_tables=None,
186 + context_lens=context_lens_tensor,
187 + block_tables=block_tables,
188 use_cuda_graph=False,
189 )
149 - return input_tokens, input_positions, input_metadata, prompt_lens
190 + return (input_tokens, input_positions, input_metadata, prompt_lens,
191 + subquery_lens)
192
193 def _prepare_decode(
194 self,
@@ -248,6 +290,9 @@ class ModelRunner:
290 input_metadata = InputMetadata(
291 is_prompt=False,
292 slot_mapping=slot_mapping,
293 + prompt_lens=None,
294 + max_seq_len=None,
295 + start_loc=None,
296 max_context_len=max_context_len,
297 context_lens=context_lens,
298 block_tables=block_tables,
@@ -259,6 +304,7 @@ class ModelRunner:
304 self,
305 seq_group_metadata_list: List[SequenceGroupMetadata],
306 prompt_lens: List[int],
307 + subquery_lens: Optional[List[int]],
308 ) -> SamplingMetadata:
309 seq_groups: List[Tuple[List[int], SamplingParams]] = []
310 selected_token_indices: List[int] = []
@@ -266,7 +312,7 @@ class ModelRunner:
312 categorized_sample_indices = {t: [] for t in SamplingType}
313 categorized_sample_indices_start_idx = 0
314
269 - max_prompt_len = max(prompt_lens) if prompt_lens else 1
315 + max_subquery_len = max(subquery_lens) if subquery_lens else 1
316 for i, seq_group_metadata in enumerate(seq_group_metadata_list):
317 seq_ids = list(seq_group_metadata.seq_data.keys())
318 sampling_params = seq_group_metadata.sampling_params
@@ -274,10 +320,11 @@ class ModelRunner:
320
321 if seq_group_metadata.is_prompt:
322 assert len(seq_ids) == 1
277 - prompt_len = prompt_lens[i]
323 + assert subquery_lens is not None
324 + subquery_len = subquery_lens[i]
325 if sampling_params.prompt_logprobs is not None:
326 # NOTE: prompt token positions do not need sample, skip
280 - categorized_sample_indices_start_idx += prompt_len - 1
327 + categorized_sample_indices_start_idx += subquery_len - 1
328
329 categorized_sample_indices[
330 sampling_params.sampling_type].append(
@@ -287,10 +334,10 @@ class ModelRunner:
334 if sampling_params.prompt_logprobs is not None:
335 selected_token_indices.extend(
336 range(selected_token_start_idx,
290 - selected_token_start_idx + prompt_len - 1))
337 + selected_token_start_idx + subquery_len - 1))
338 selected_token_indices.append(selected_token_start_idx +
292 - prompt_len - 1)
293 - selected_token_start_idx += max_prompt_len
339 + subquery_len - 1)
340 + selected_token_start_idx += max_subquery_len
341 else:
342 num_seqs = len(seq_ids)
343 selected_token_indices.extend(
@@ -335,14 +382,16 @@ class ModelRunner:
382 is_prompt = seq_group_metadata_list[0].is_prompt
383 # Prepare input tensors.
384 if is_prompt:
338 - (input_tokens, input_positions, input_metadata,
339 - prompt_lens) = self._prepare_prompt(seq_group_metadata_list)
385 + (input_tokens, input_positions, input_metadata, prompt_lens,
386 + subquery_lens) = self._prepare_prompt(seq_group_metadata_list)
387 else:
388 (input_tokens, input_positions, input_metadata
389 ) = self._prepare_decode(seq_group_metadata_list)
390 + subquery_lens = None
391 prompt_lens = []
392 sampling_metadata = self._prepare_sample(seq_group_metadata_list,
345 - prompt_lens)
393 + prompt_lens,
394 + subquery_lens)
395
396 def get_size_or_none(x: Optional[torch.Tensor]):
397 return x.size() if x is not None else None
@@ -359,6 +408,12 @@ class ModelRunner:
408 input_metadata.is_prompt,
409 "slot_mapping_size":
410 get_size_or_none(input_metadata.slot_mapping),
411 + "prompt_lens_size":
412 + get_size_or_none(input_metadata.prompt_lens),
413 + "max_seq_len":
414 + input_metadata.max_seq_len,
415 + "start_loc_size":
416 + get_size_or_none(input_metadata.start_loc),
417 "max_context_len":
418 input_metadata.max_context_len,
419 "context_lens_size":
@@ -376,6 +431,10 @@ class ModelRunner:
431 broadcast(input_positions, src=0)
432 if input_metadata.slot_mapping is not None:
433 broadcast(input_metadata.slot_mapping, src=0)
434 + if input_metadata.prompt_lens is not None:
435 + broadcast(input_metadata.prompt_lens, src=0)
436 + if input_metadata.start_loc is not None:
437 + broadcast(input_metadata.start_loc, src=0)
438 if input_metadata.context_lens is not None:
439 broadcast(input_metadata.context_lens, src=0)
440 if input_metadata.block_tables is not None:
@@ -400,6 +459,20 @@ class ModelRunner:
459 broadcast(slot_mapping, src=0)
460 else:
461 slot_mapping = None
462 + if py_data["prompt_lens_size"] is not None:
463 + prompt_lens = torch.empty(*py_data["prompt_lens_size"],
464 + dtype=torch.long,
465 + device="cuda")
466 + broadcast(prompt_lens, src=0)
467 + else:
468 + prompt_lens = None
469 + if py_data["start_loc_size"] is not None:
470 + start_loc = torch.empty(*py_data["start_loc_size"],
471 + dtype=torch.long,
472 + device="cuda")
473 + broadcast(start_loc, src=0)
474 + else:
475 + start_loc = None
476 if py_data["context_lens_size"] is not None:
477 context_lens = torch.empty(*py_data["context_lens_size"],
478 dtype=torch.int,
@@ -422,6 +495,9 @@ class ModelRunner:
495 input_metadata = InputMetadata(
496 is_prompt=py_data["is_prompt"],
497 slot_mapping=slot_mapping,
498 + prompt_lens=prompt_lens,
499 + max_seq_len=py_data["max_seq_len"],
500 + start_loc=start_loc,
501 max_context_len=py_data["max_context_len"],
502 context_lens=context_lens,
503 block_tables=block_tables,
@@ -534,6 +610,9 @@ class ModelRunner:
610 input_metadata = InputMetadata(
611 is_prompt=False,
612 slot_mapping=slot_mapping[:batch_size],
613 + prompt_lens=None,
614 + max_seq_len=None,
615 + start_loc=None,
616 max_context_len=self.max_context_len_to_capture,
617 context_lens=context_lens[:batch_size],
618 block_tables=block_tables[:batch_size],