@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d4f3985907
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Core] Sliding window for block manager v2 (#4545)

Co-authored-by: Ruth Evans <ruthevans@Ruths-MacBook-Pro.local>

Michał Moskal committed May 27, 2024 at 19:07 UTC d4f398590786f0015d474b03a3d078db1e7d1be2
12 files changed +457 -45
tests/core/block/e2e/conftest.py
+26
@@ -1,3 +1,5 @@
1 +from typing import Callable, Iterable, Optional
2 +
3 import pytest
4
5 from vllm import LLM
@@ -40,3 +42,27 @@ def create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
42 for llm in generator_inner():
43 yield llm
44 del llm
45 +
46 +
47 +def get_text_from_llm_generator(llm_generator: Iterable[LLM],
48 + prompts,
49 + sampling_params,
50 + llm_cb: Optional[Callable[[LLM],
51 + None]] = None):
52 + for llm in llm_generator:
53 + if llm_cb:
54 + llm_cb(llm)
55 + outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
56 + text = [output.outputs[0].text for output in outputs]
57 + del llm
58 +
59 + return text
60 +
61 +
62 +def get_token_ids_from_llm_generator(llm_generator, prompts, sampling_params):
63 + for llm in llm_generator:
64 + outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
65 + token_ids = [output.outputs[0].token_ids for output in outputs]
66 + del llm
67 +
68 + return token_ids
tests/core/block/e2e/test_correctness.py
+2 -9
@@ -4,6 +4,8 @@ import pytest
4
5 from vllm import SamplingParams
6
7 +from .conftest import get_token_ids_from_llm_generator
8 +
9
10 @pytest.mark.parametrize(
11 "common_llm_kwargs",
@@ -444,12 +446,3 @@ def test_auto_prefix_caching_with_preemption(baseline_llm_generator,
446 assert expected_token_ids == actual_token_ids
447
448 assert baseline_token_ids == test_token_ids
447 -
448 -
449 -def get_token_ids_from_llm_generator(llm_generator, prompts, sampling_params):
450 - for llm in llm_generator:
451 - outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
452 - token_ids = [output.outputs[0].token_ids for output in outputs]
453 - del llm
454 -
455 - return token_ids
tests/core/block/e2e/test_correctness_sliding_window.py new
+168
@@ -0,0 +1,168 @@
1 +import random
2 +from typing import List
3 +
4 +import pytest
5 +
6 +from vllm import LLM, SamplingParams
7 +
8 +from .conftest import get_text_from_llm_generator
9 +
10 +# relatively small model with 4k sliding window
11 +MODEL = "bigcode/starcoder2-3b"
12 +BLOCK_SIZE = 16
13 +
14 +
15 +@pytest.mark.parametrize(
16 + "common_llm_kwargs",
17 + [{
18 + "model": MODEL,
19 +
20 + # skip cuda graph creation for fast test.
21 + "enforce_eager": True,
22 + "block_size": BLOCK_SIZE,
23 + # needed due to https://github.com/vllm-project/vllm/issues/1908#issuecomment-2101122008
24 + "num_gpu_blocks_override": 100000 // BLOCK_SIZE,
25 + }])
26 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
27 +@pytest.mark.parametrize("baseline_llm_kwargs", [{
28 + "use_v2_block_manager": False
29 +}])
30 +@pytest.mark.parametrize("test_llm_kwargs", [{"use_v2_block_manager": True}])
31 +@pytest.mark.parametrize("batch_size", [5])
32 +@pytest.mark.parametrize("seed", [1])
33 +def test_sliding_window_retrival(baseline_llm_generator, test_llm_generator,
34 + batch_size, seed):
35 + """
36 + The test does a bunch of assignments "x1 = 10\nx2 = 33\n..." and then
37 + asks for value of one of them (which is outside the sliding window).
38 + If we tell it upfront which we are going to be looking for, then
39 + it answers correctly (mostly).
40 +
41 + Additionally, we compare the results of the v1 and v2 managers.
42 + """
43 + sampling_params = SamplingParams(
44 + max_tokens=1024,
45 + ignore_eos=True,
46 + temperature=0.0,
47 + )
48 +
49 + prompts, answer, indices = prep_prompts(batch_size)
50 +
51 + print('Getting token ids from block manager v1')
52 + baseline_texts = get_text_from_llm_generator(baseline_llm_generator,
53 + prompts,
54 + sampling_params,
55 + llm_cb=check_window(prompts))
56 +
57 + check_answers(indices, answer, baseline_texts)
58 +
59 + print('Getting token ids from block manager v2')
60 + test_texts = get_text_from_llm_generator(test_llm_generator, prompts,
61 + sampling_params)
62 + check_answers(indices, answer, test_texts)
63 +
64 + cmp = [
65 + expected_text == actual_text
66 + for expected_text, actual_text in zip(baseline_texts, test_texts)
67 + ]
68 + print(cmp)
69 + # make sure it's mostly OK; this is possibly because https://github.com/vllm-project/vllm/pull/4768
70 + # however, https://github.com/vllm-project/vllm/issues/3385#issuecomment-1995924290
71 + # states that xformers and flash_attn have different ideas about the window
72 + # size anyways
73 + assert sum(cmp) > 0.7 * len(cmp)
74 +
75 +
76 +@pytest.mark.parametrize(
77 + "common_llm_kwargs",
78 + [{
79 + "model": MODEL,
80 +
81 + # skip cuda graph creation for fast test.
82 + "enforce_eager": True,
83 + "block_size": BLOCK_SIZE,
84 + "num_gpu_blocks_override": 100000 // BLOCK_SIZE,
85 + }])
86 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
87 +@pytest.mark.parametrize("test_llm_kwargs", [{
88 + "use_v2_block_manager": True,
89 + "enable_chunked_prefill": True
90 +}])
91 +@pytest.mark.parametrize("batch_size", [5])
92 +@pytest.mark.parametrize("seed", [1])
93 +def test_sliding_window_chunked_prefill(test_llm_generator, batch_size, seed):
94 + """
95 + This is similar to test_sliding_window_retrival, however, it doesn't
96 + compare against the v1 block manager since v1 doesn't support
97 + chunked prefill with sliding window.
98 +
99 + The results with and without chunked prefill are not the same due to
100 + numerical instabilities.
101 + """
102 + sampling_params = SamplingParams(
103 + max_tokens=10,
104 + ignore_eos=True,
105 + temperature=0.0,
106 + )
107 +
108 + prompts, answer, indices = prep_prompts(batch_size)
109 +
110 + # We don't compare with the baseline model here, since the results
111 + # slightly different due to different tailing in attention.
112 + test_texts = get_text_from_llm_generator(test_llm_generator,
113 + prompts,
114 + sampling_params,
115 + llm_cb=check_window(prompts))
116 + check_answers(indices, answer, test_texts)
117 +
118 +
119 +def prep_prompts(batch_size: int):
120 + """
121 + Generate prompts which a bunch of assignments,
122 + then asking for the value of one of them.
123 + The prompt is just under 10k tokens; sliding window is 4k
124 + so the answer is outside sliding window, but should still be correct.
125 + """
126 + prompts: List[str] = []
127 + answer: List[int] = []
128 + indices: List[int] = []
129 + random.seed(1)
130 + for _ in range(batch_size):
131 + idx = random.randint(30, 90)
132 + indices.append(idx)
133 + prompt = "```python\n# We set a number of variables, " + \
134 + f"x{idx} will be important later\n"
135 + ln = random.randint(800, 1100)
136 + for k in range(30, ln):
137 + v = random.randint(10, 99)
138 + if k == idx:
139 + answer.append(v)
140 + prompt += f"x{k} = {v}\n"
141 + prompt += f"# Now, we check the value of x{idx}:\n"
142 + prompt += f"assert x{idx} == "
143 + prompts.append(prompt)
144 + return prompts, answer, indices
145 +
146 +
147 +def check_answers(indices: List[int], answer: List[int], outputs: List[str]):
148 + answer2 = [int(text[0:2].strip()) for text in outputs]
149 + print(list(zip(indices, zip(answer, answer2))))
150 + numok = 0
151 + for a1, a2 in zip(answer, answer2):
152 + if a1 == a2:
153 + numok += 1
154 + frac_ok = numok / len(answer)
155 + print(f"Num OK: {numok}/{len(answer)} {frac_ok}")
156 + assert frac_ok > 0.7
157 +
158 +
159 +def check_window(prompts: List[str]):
160 +
161 + def inner(llm: LLM):
162 + sliding_window = llm.llm_engine.model_config.get_sliding_window()
163 + assert sliding_window and sliding_window > 0
164 + assert any(
165 + len(llm.get_tokenizer().tokenize(prompt)) > sliding_window
166 + for prompt in prompts)
167 +
168 + return inner
tests/core/block/test_block_manager_v2.py
+69
@@ -101,3 +101,72 @@ def test_append_slots(block_size, prompt_len, num_slots_to_append,
101 range(prompt_len + num_slots_to_append + num_lookahead_slots)),
102 block_size)) - len(chunk_list(list(range(prompt_len)), block_size))
103 assert num_consumed_blocks == expected_consumed_blocks
104 +
105 +
106 +@pytest.mark.parametrize("block_size", [8, 16])
107 +@pytest.mark.parametrize("prompt_len", [10, 300, 1000])
108 +@pytest.mark.parametrize("num_slots_to_append", [50])
109 +@pytest.mark.parametrize("sliding_window", [20, 32, 200, 512])
110 +def test_sliding_window(block_size, prompt_len, num_slots_to_append,
111 + sliding_window):
112 + """Verify append_slots consumes the correct number of blocks from the block
113 + table.
114 + """
115 +
116 + num_gpu_blocks = 1024
117 + watermark = 0.1
118 + block_manager = BlockSpaceManagerV2(
119 + block_size=block_size,
120 + num_gpu_blocks=num_gpu_blocks,
121 + num_cpu_blocks=0,
122 + watermark=watermark,
123 + sliding_window=sliding_window,
124 + )
125 +
126 + def check_used(min_n, max_n=None):
127 + if max_n is None:
128 + max_n = min_n
129 + used = num_gpu_blocks - block_manager.get_num_free_gpu_blocks()
130 + #print("check", min_n, used, max_n)
131 + assert min_n <= used
132 + assert used <= max_n
133 +
134 + def num_blocks(num_tokens):
135 + return (num_tokens + block_size - 1) // block_size
136 +
137 + check_used(0)
138 +
139 + seq_group = create_seq_group(
140 + seq_prompt_len=prompt_len,
141 + seq_output_lens=[0],
142 + )
143 +
144 + check_used(0)
145 +
146 + # Allocate seq
147 + assert block_manager.can_allocate(seq_group)
148 + block_manager.allocate(seq_group)
149 +
150 + check_used(num_blocks(prompt_len))
151 +
152 + # Seq seq to RUNNING
153 + seq = seq_group.get_seqs()[0]
154 + seq.status = SequenceStatus.RUNNING
155 +
156 + seq.data.update_num_computed_tokens(prompt_len)
157 + check_used(num_blocks(prompt_len))
158 +
159 + # this is how we compute it in BlockSpaceManagerV2.__init__
160 + sliding_blocks = (sliding_window // block_size) + 2
161 + # plus one block for null block
162 + sliding_blocks += 1
163 +
164 + # Append tokens to the sequeqnce
165 + for token_id in range(num_slots_to_append):
166 + seq.append_token_id(token_id, {token_id: Logprob(0.0)})
167 + seq.data.update_num_computed_tokens(1)
168 + block_manager.append_slots(seq, num_lookahead_slots=0)
169 + if prompt_len < sliding_window + 10:
170 + check_used(0, sliding_blocks + 1)
171 + else:
172 + check_used(sliding_blocks, sliding_blocks + 1)
vllm/attention/ops/prefix_prefill.py
+5 -1
@@ -697,6 +697,10 @@ if triton.__version__ >= "2.1.0":
697
698 grid = (batch, head, triton.cdiv(max_input_len, BLOCK)) # batch, head,
699
700 + # 0 means "disable"
701 + if sliding_window is None or sliding_window <= 0:
702 + sliding_window = 0
703 +
704 num_warps = 8 if Lk <= 64 else 8
705 if alibi_slopes is not None:
706 _fwd_kernel_alibi[grid](
@@ -794,7 +798,7 @@ if triton.__version__ >= "2.1.0":
798 BLOCK_DMODEL=Lk,
799 BLOCK_DMODEL_PADDED=Lk_padded,
800 BLOCK_N=BLOCK,
797 - SLIDING_WINDOW=sliding_window if sliding_window is not None else 0,
801 + SLIDING_WINDOW=sliding_window,
802 num_warps=num_warps,
803 num_stages=1,
804 )
vllm/core/block/block_table.py
+32 -2
@@ -20,6 +20,10 @@ class BlockTable:
20 _blocks (Optional[List[Block]], optional): An optional list of existing
21 blocks to initialize the BlockTable with. If not provided, an empty
22 BlockTable is created.
23 + max_block_sliding_window (Optional[int], optional): The number of
24 + blocks to keep around for each sequance. If None, all blocks
25 + are kept (eg., when sliding window is not used).
26 + It should at least fit the sliding window size of the model.
27
28 Attributes:
29 _block_size (int): The maximum number of tokens that can be stored in a
@@ -37,6 +41,7 @@ class BlockTable:
41 block_size: int,
42 block_allocator: DeviceAwareBlockAllocator,
43 _blocks: Optional[List[Block]] = None,
44 + max_block_sliding_window: Optional[int] = None,
45 ):
46 self._block_size = block_size
47 self._allocator = block_allocator
@@ -44,6 +49,7 @@ class BlockTable:
49 _blocks = []
50 self._blocks: List[Block] = _blocks
51
52 + self._max_block_sliding_window = max_block_sliding_window
53 # Use helper method instead of directly calculating, as blocks
54 # may not be allocated.
55 self._num_full_slots = len(self._get_all_token_ids())
@@ -89,7 +95,8 @@ class BlockTable:
95
96 def append_token_ids(self,
97 token_ids: List[int],
92 - num_lookahead_slots: int = 0) -> None:
98 + num_lookahead_slots: int = 0,
99 + num_computed_slots: Optional[int] = None) -> None:
100 """Appends a sequence of token IDs to the existing blocks in the
101 BlockTable.
102
@@ -104,13 +111,35 @@ class BlockTable:
111
112 Args:
113 token_ids (List[int]): The sequence of token IDs to be appended.
114 + num_computed_slots (Optional[int]): The number of KV cache slots
115 + that are already filled (computed).
116 + When sliding window is enabled, this is used to compute how many
117 + blocks to drop at the front of the sequence.
118 + Without sliding window, None can be passed.
119 + Without chunked prefill, it should be the same as
120 + _num_full_slots.
121 """
108 - assert self._is_allocated
122 + assert self._is_allocated, "no blocks have been allocated"
123 assert len(self._blocks) > 0
124
125 + # Drop blocks that are no longer needed due to sliding window
126 + if self._max_block_sliding_window is not None:
127 + null_block = self._allocator.allocate_or_get_null_block()
128 + assert num_computed_slots is not None
129 + end_block_idx = (num_computed_slots //
130 + self._block_size) - self._max_block_sliding_window
131 + for idx in range(0, end_block_idx):
132 + b = self._blocks[idx]
133 + if b is not null_block:
134 + self._allocator.free(b)
135 + self._blocks[idx] = null_block
136 +
137 + # Ensure there are enough empty slots for the new tokens plus
138 + # lookahead slots
139 self.ensure_num_empty_slots(num_empty_slots=len(token_ids) +
140 num_lookahead_slots)
141
142 + # Update the blocks with the new tokens
143 blocks = self._blocks[self._num_full_slots // self._block_size:]
144 token_blocks = self._chunk_token_blocks_for_append(token_ids)
145
@@ -168,6 +197,7 @@ class BlockTable:
197 block_size=self._block_size,
198 block_allocator=self._allocator,
199 _blocks=forked_blocks,
200 + max_block_sliding_window=self._max_block_sliding_window,
201 )
202
203 def free(self) -> None:
vllm/core/block/cpu_gpu_block_allocator.py
+74
@@ -105,11 +105,19 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
105 Device.GPU: gpu_block_allocator,
106 }
107
108 + self._null_block: Optional[Block] = None
109 +
110 self._block_ids_to_allocator: Dict[int, BlockAllocator] = {}
111 for _, allocator in self._allocators.items():
112 for block_id in allocator.all_block_ids:
113 self._block_ids_to_allocator[block_id] = allocator
114
115 + def allocate_or_get_null_block(self) -> Block:
116 + if self._null_block is None:
117 + self._null_block = NullBlock(
118 + self.allocate_mutable(None, Device.GPU))
119 + return self._null_block
120 +
121 def allocate_mutable(self, prev_block: Optional[Block],
122 device: Device) -> Block:
123 """Allocates a new mutable block on the specified device.
@@ -149,6 +157,9 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
157 Args:
158 block (Block): The block to be freed.
159 """
160 + # Null block should never be freed
161 + if isinstance(block, NullBlock):
162 + return
163 block_id = block.block_id
164 assert block_id is not None
165 allocator = self._block_ids_to_allocator[block_id]
@@ -165,6 +176,8 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
176 List[Block]: A new list of blocks that shares the same memory as the
177 original sequence.
178 """
179 + # do not attempt to fork the null block
180 + assert not isinstance(last_block, NullBlock)
181 block_id = last_block.block_id
182 assert block_id is not None
183 allocator = self._block_ids_to_allocator[block_id]
@@ -226,3 +239,64 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
239
240 def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
241 raise NotImplementedError
242 +
243 +
244 +class NullBlock(Block):
245 + """
246 + Null blocks are used as a placeholders for KV cache blocks that have
247 + been dropped due to sliding window.
248 + This implementation just wraps an ordinary block and prevents it from
249 + being modified. It also allows for testing if a block is NullBlock
250 + via isinstance().
251 + """
252 +
253 + def __init__(self, proxy: Block):
254 + super().__init__()
255 + self._proxy = proxy
256 +
257 + def append_token_ids(self, token_ids: List[BlockId]):
258 + raise ValueError("null block should not be modified")
259 +
260 + @property
261 + def block_id(self):
262 + return self._proxy.block_id
263 +
264 + @block_id.setter
265 + def block_id(self, value: Optional[BlockId]):
266 + raise ValueError("null block should not be modified")
267 +
268 + @property
269 + def token_ids(self) -> List[BlockId]:
270 + return self._proxy.token_ids
271 +
272 + @property
273 + def num_empty_slots(self) -> BlockId:
274 + return self._proxy.num_empty_slots
275 +
276 + @property
277 + def is_full(self):
278 + return self._proxy.is_full
279 +
280 + @property
281 + def prev_block(self):
282 + return self._proxy.prev_block
283 +
284 + @property
285 + def computed(self):
286 + return self._proxy.computed
287 +
288 + @computed.setter
289 + def computed(self, value):
290 + self._proxy.computed = value
291 +
292 + @property
293 + def last_accessed(self) -> float:
294 + return self._proxy.last_accessed
295 +
296 + @last_accessed.setter
297 + def last_accessed(self, last_accessed_ts: float):
298 + self._proxy.last_accessed = last_accessed_ts
299 +
300 + @property
301 + def content_hash(self):
302 + return self._proxy.content_hash
vllm/core/block/interfaces.py
+9
@@ -203,3 +203,12 @@ class DeviceAwareBlockAllocator(ABC):
203 def get_common_computed_block_ids(
204 self, seq_block_ids: List[List[int]]) -> List[int]:
205 pass
206 +
207 + @abstractmethod
208 + def allocate_or_get_null_block(self) -> Block:
209 + """
210 + Null blocks are used as a placeholders for KV cache blocks that have
211 + been dropped due to sliding window.
212 + There is at most one null block per allocator.
213 + """
214 + pass
vllm/core/block_manager_v2.py
+17 -7
@@ -66,9 +66,18 @@ class BlockSpaceManagerV2(BlockSpaceManager):
66 self.num_total_gpu_blocks = num_gpu_blocks
67 self.num_total_cpu_blocks = num_cpu_blocks
68
69 - assert sliding_window is None, "Sliding window not yet supported"
70 -
71 - self.block_sliding_window = None
69 + self.sliding_window = sliding_window
70 + # max_block_sliding_window is the max number of blocks that need to be
71 + # allocated
72 + self.max_block_sliding_window = None
73 + if sliding_window is not None:
74 + # +1 here because // rounds down
75 + num_blocks = sliding_window // block_size + 1
76 + # +1 here because the last block may not be full,
77 + # and so the sequence stretches one more block at the beginning
78 + # For example, if sliding_window is 3 and block_size is 4,
79 + # we may need 2 blocks when the second block only holds 1 token.
80 + self.max_block_sliding_window = num_blocks + 1
81
82 self.watermark = watermark
83 assert watermark >= 0.0
@@ -96,10 +105,9 @@ class BlockSpaceManagerV2(BlockSpaceManager):
105 block_size=self.block_size,
106 )
107
99 - assert self.block_sliding_window is None
100 - if self.block_sliding_window is not None:
108 + if self.max_block_sliding_window is not None:
109 num_required_blocks = min(num_required_blocks,
102 - self.block_sliding_window)
110 + self.max_block_sliding_window)
111
112 num_free_gpu_blocks = self.block_allocator.get_num_free_blocks(
113 device=Device.GPU)
@@ -125,8 +133,9 @@ class BlockSpaceManagerV2(BlockSpaceManager):
133 block_table = BlockTable(
134 block_size=self.block_size,
135 block_allocator=self.block_allocator,
136 + max_block_sliding_window=self.max_block_sliding_window,
137 )
129 - assert self.block_sliding_window is None
138 +
139 block_table.allocate(seq.get_token_ids())
140 self.block_tables[seq.seq_id] = block_table
141
@@ -174,6 +183,7 @@ class BlockSpaceManagerV2(BlockSpaceManager):
183 block_table.append_token_ids(
184 token_ids=block_table.get_unseen_token_ids(seq.get_token_ids()),
185 num_lookahead_slots=num_lookahead_slots,
186 + num_computed_slots=seq.data.get_num_computed_tokens(),
187 )
188
189 # Return any new copy-on-writes.
vllm/engine/arg_utils.py
+2 -1
@@ -648,7 +648,8 @@ class EngineArgs:
648 guided_decoding_backend=self.guided_decoding_backend)
649
650 if (model_config.get_sliding_window() is not None
651 - and scheduler_config.chunked_prefill_enabled):
651 + and scheduler_config.chunked_prefill_enabled
652 + and not scheduler_config.use_v2_block_manager):
653 raise ValueError(
654 "Chunked prefill is not supported with sliding window. "
655 "Set --disable-sliding-window to disable sliding window.")
vllm/worker/cache_engine.py
+4 -1
@@ -68,8 +68,11 @@ class CacheEngine:
68 pin_memory = is_pin_memory_available() if device == "cpu" else False
69 kv_cache: List[torch.Tensor] = []
70 for _ in range(self.num_layers):
71 + # null block in CpuGpuBlockAllocator requires at least that
72 + # block to be zeroed-out.
73 + # We zero-out everything for simplicity.
74 kv_cache.append(
72 - torch.empty(kv_cache_shape,
75 + torch.zeros(kv_cache_shape,
76 dtype=self.dtype,
77 pin_memory=pin_memory,
78 device=device))
vllm/worker/model_runner.py
+49 -24
@@ -269,6 +269,12 @@ class ModelRunner:
269 if len(seq_group_metadata_list) == 0:
270 return ModelInput.empty(self.device)
271
272 + if self.sliding_window is not None:
273 + sliding_window_blocks = (self.sliding_window + self.block_size -
274 + 1) // self.block_size
275 + block_aligned_sliding_window = \
276 + sliding_window_blocks * self.block_size
277 +
278 for seq_group_metadata in seq_group_metadata_list:
279 seq_ids = list(seq_group_metadata.seq_data.keys())
280 is_prompt = seq_group_metadata.is_prompt
@@ -309,6 +315,30 @@ class ModelRunner:
315 and self.sliding_window is None
316 and is_prompt)
317
318 + # These are seq_len/context_len capped to the sliding window.
319 + # They are passed to decode kernel.
320 + # We still need original seq_len/context_len to compute slot
321 + # mapping (and input position) below.
322 + curr_sliding_window_blocks = None
323 + sliding_seq_len = seq_len
324 + sliding_context_len = context_len
325 +
326 + # TODO(sang): This is a hack to make sliding window work with
327 + # paged attn. We can remove it if we make paged attn kernel
328 + # to properly handle slinding window attn.
329 + if (self.sliding_window is not None and not is_prompt):
330 + curr_sliding_window_blocks = sliding_window_blocks
331 + if self.scheduler_config.use_v2_block_manager:
332 + # number of elements in last block
333 + suff_len = seq_len % self.block_size
334 + sliding_seq_len = min(
335 + seq_len, block_aligned_sliding_window + suff_len)
336 + if suff_len > 0:
337 + curr_sliding_window_blocks += 1
338 + else:
339 + sliding_seq_len = min(seq_len, self.sliding_window)
340 + sliding_context_len = sliding_seq_len - 1
341 +
342 # TODO(sang): Combine chunked prefill and prefix caching by
343 # only allowing multiple of block_size chunk size.
344 # NOTE: This only works for oooooooxxx style attention.
@@ -316,6 +346,13 @@ class ModelRunner:
346 assert computed_block_nums is not None
347 context_len = len(computed_block_nums) * self.block_size
348 tokens = tokens[context_len:]
349 +
350 + # need to think what to set it to when we have both sliding
351 + # window and prefix caching...
352 + assert self.sliding_window is None, \
353 + "Prefix caching is not supported with sliding window"
354 + sliding_context_len = context_len
355 +
356 if self.attn_backend.get_name() == "flash-attn":
357 # NOTE(woosuk): For flash-attn, the block table should
358 # include the entries for the incoming prefill tokens.
@@ -329,14 +366,9 @@ class ModelRunner:
366 if seq_group_metadata.block_tables is not None:
367 # chunked prefill or decode
368 block_table = seq_group_metadata.block_tables[seq_id]
332 - if self.sliding_window is not None:
333 - # chunked prefill doesn't support sliding window.
334 - assert (not self.scheduler_config.
335 - chunked_prefill_enabled)
336 - sliding_window_blocks = (self.sliding_window //
337 - self.block_size)
338 - block_table = block_table[-sliding_window_blocks:]
339 -
369 + if curr_sliding_window_blocks is not None:
370 + block_table = block_table[
371 + -curr_sliding_window_blocks:]
372 if self.attn_backend.get_name() == "flashinfer":
373 paged_kv_indices.extend(block_table)
374 paged_kv_indptr.append(paged_kv_indptr[-1] +
@@ -354,16 +386,9 @@ class ModelRunner:
386 block_table = []
387 block_tables.append(block_table)
388
357 - # TODO(sang): This is a hack to make sliding window work with
358 - # paged attn. We can remove it if we make paged attn kernel
359 - # to properly handle slinding window attn.
360 - if (self.sliding_window is not None and not is_prompt):
361 - seq_len = min(seq_len, self.sliding_window)
362 - context_len = seq_len - 1
363 -
364 - seq_lens.append(seq_len)
365 - context_lens.append(context_len)
366 - query_len = seq_len - context_len
389 + seq_lens.append(sliding_seq_len)
390 + context_lens.append(sliding_context_len)
391 + query_len = sliding_seq_len - sliding_context_len
392 query_lens.append(query_len)
393 input_tokens.extend(tokens)
394 input_positions.extend(list(range(context_len, seq_len)))
@@ -380,16 +405,15 @@ class ModelRunner:
405 "seq_len: {}, context_len: {}, query_len: {}".format(
406 seq_len, context_len, query_len))
407 num_decode_tokens += query_len
383 - decode_seq_lens.append(seq_len)
408 + decode_seq_lens.append(sliding_seq_len)
409
410 if lora_id > 0:
411 lora_requests.add(seq_group_metadata.lora_request)
412
388 - lora_index_mapping += [lora_id] * (seq_len - context_len)
413 + lora_index_mapping += [lora_id] * query_len
414 lora_prompt_mapping.extend(
415 [lora_id] *
391 - (seq_len -
392 - context_len if seq_group_metadata.sampling_params
416 + (query_len if seq_group_metadata.sampling_params
417 and seq_group_metadata.sampling_params.prompt_logprobs
418 else 1))
419
@@ -417,9 +441,10 @@ class ModelRunner:
441 start_idx = 0
442 if self.sliding_window is not None:
443 if is_prompt:
420 - assert context_len == 0, (
444 + assert self.scheduler_config.use_v2_block_manager \
445 + or context_len == 0, (
446 "Prefix caching is currently not supported with "
422 - "sliding window attention")
447 + "sliding window attention in V1 block manager")
448 # It is an optimization. When it is decoding, it is always
449 # 0. When prefill, we use it to not write slots to kv cache
450 # to save memory.