A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Core] Sliding window for block manager v2 (#4545)
Co-authored-by: Ruth Evans <ruthevans@Ruths-MacBook-Pro.local>
Michał Moskal committed
May 27, 2024 at 19:07 UTC
d4f398590786f0015d474b03a3d078db1e7d1be2
12 files changed
+457
-45
tests/core/block/e2e/conftest.py
+26
@@ -1,3 +1,5 @@
1
+from typing import Callable, Iterable, Optional
2
+
3
import pytest
4
5
from vllm import LLM
@@ -40,3 +42,27 @@ def create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
42
for llm in generator_inner():
43
yield llm
44
del llm
45
+
46
+
47
+def get_text_from_llm_generator(llm_generator: Iterable[LLM],
48
+ prompts,
49
+ sampling_params,
50
+ llm_cb: Optional[Callable[[LLM],
51
+ None]] = None):
52
+ for llm in llm_generator:
53
+ if llm_cb:
54
+ llm_cb(llm)
55
+ outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
56
+ text = [output.outputs[0].text for output in outputs]
57
+ del llm
58
+
59
+ return text
60
+
61
+
62
+def get_token_ids_from_llm_generator(llm_generator, prompts, sampling_params):
63
+ for llm in llm_generator:
64
+ outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
65
+ token_ids = [output.outputs[0].token_ids for output in outputs]
66
+ del llm
67
+
68
+ return token_ids
tests/core/block/e2e/test_correctness.py
+2
-9
@@ -4,6 +4,8 @@ import pytest
4
5
from vllm import SamplingParams
6
7
+from .conftest import get_token_ids_from_llm_generator
8
+
9
10
@pytest.mark.parametrize(
11
"common_llm_kwargs",
@@ -444,12 +446,3 @@ def test_auto_prefix_caching_with_preemption(baseline_llm_generator,
446
assert expected_token_ids == actual_token_ids
447
448
assert baseline_token_ids == test_token_ids
447
-
448
-
449
-def get_token_ids_from_llm_generator(llm_generator, prompts, sampling_params):
450
- for llm in llm_generator:
451
- outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
452
- token_ids = [output.outputs[0].token_ids for output in outputs]
453
- del llm
454
-
455
- return token_ids
tests/core/block/e2e/test_correctness_sliding_window.py
new
+168
@@ -0,0 +1,168 @@
1
+import random
2
+from typing import List
3
+
4
+import pytest
5
+
6
+from vllm import LLM, SamplingParams
7
+
8
+from .conftest import get_text_from_llm_generator
9
+
10
+# relatively small model with 4k sliding window
11
+MODEL = "bigcode/starcoder2-3b"
12
+BLOCK_SIZE = 16
13
+
14
+
15
+@pytest.mark.parametrize(
16
+ "common_llm_kwargs",
17
+ [{
18
+ "model": MODEL,
19
+
20
+ # skip cuda graph creation for fast test.
21
+ "enforce_eager": True,
22
+ "block_size": BLOCK_SIZE,
23
+ # needed due to https://github.com/vllm-project/vllm/issues/1908#issuecomment-2101122008
24
+ "num_gpu_blocks_override": 100000 // BLOCK_SIZE,
25
+ }])
26
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
27
+@pytest.mark.parametrize("baseline_llm_kwargs", [{
28
+ "use_v2_block_manager": False
29
+}])
30
+@pytest.mark.parametrize("test_llm_kwargs", [{"use_v2_block_manager": True}])
31
+@pytest.mark.parametrize("batch_size", [5])
32
+@pytest.mark.parametrize("seed", [1])
33
+def test_sliding_window_retrival(baseline_llm_generator, test_llm_generator,
34
+ batch_size, seed):
35
+ """
36
+ The test does a bunch of assignments "x1 = 10\nx2 = 33\n..." and then
37
+ asks for value of one of them (which is outside the sliding window).
38
+ If we tell it upfront which we are going to be looking for, then
39
+ it answers correctly (mostly).
40
+
41
+ Additionally, we compare the results of the v1 and v2 managers.
42
+ """
43
+ sampling_params = SamplingParams(
44
+ max_tokens=1024,
45
+ ignore_eos=True,
46
+ temperature=0.0,
47
+ )
48
+
49
+ prompts, answer, indices = prep_prompts(batch_size)
50
+
51
+ print('Getting token ids from block manager v1')
52
+ baseline_texts = get_text_from_llm_generator(baseline_llm_generator,
53
+ prompts,
54
+ sampling_params,
55
+ llm_cb=check_window(prompts))
56
+
57
+ check_answers(indices, answer, baseline_texts)
58
+
59
+ print('Getting token ids from block manager v2')
60
+ test_texts = get_text_from_llm_generator(test_llm_generator, prompts,
61
+ sampling_params)
62
+ check_answers(indices, answer, test_texts)
63
+
64
+ cmp = [
65
+ expected_text == actual_text
66
+ for expected_text, actual_text in zip(baseline_texts, test_texts)
67
+ ]
68
+ print(cmp)
69
+ # make sure it's mostly OK; this is possibly because https://github.com/vllm-project/vllm/pull/4768
70
+ # however, https://github.com/vllm-project/vllm/issues/3385#issuecomment-1995924290
71
+ # states that xformers and flash_attn have different ideas about the window
72
+ # size anyways
73
+ assert sum(cmp) > 0.7 * len(cmp)
74
+
75
+
76
+@pytest.mark.parametrize(
77
+ "common_llm_kwargs",
78
+ [{
79
+ "model": MODEL,
80
+
81
+ # skip cuda graph creation for fast test.
82
+ "enforce_eager": True,
83
+ "block_size": BLOCK_SIZE,
84
+ "num_gpu_blocks_override": 100000 // BLOCK_SIZE,
85
+ }])
86
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
87
+@pytest.mark.parametrize("test_llm_kwargs", [{
88
+ "use_v2_block_manager": True,
89
+ "enable_chunked_prefill": True
90
+}])
91
+@pytest.mark.parametrize("batch_size", [5])
92
+@pytest.mark.parametrize("seed", [1])
93
+def test_sliding_window_chunked_prefill(test_llm_generator, batch_size, seed):
94
+ """
95
+ This is similar to test_sliding_window_retrival, however, it doesn't
96
+ compare against the v1 block manager since v1 doesn't support
97
+ chunked prefill with sliding window.
98
+
99
+ The results with and without chunked prefill are not the same due to
100
+ numerical instabilities.
101
+ """
102
+ sampling_params = SamplingParams(
103
+ max_tokens=10,
104
+ ignore_eos=True,
105
+ temperature=0.0,
106
+ )
107
+
108
+ prompts, answer, indices = prep_prompts(batch_size)
109
+
110
+ # We don't compare with the baseline model here, since the results
111
+ # slightly different due to different tailing in attention.
112
+ test_texts = get_text_from_llm_generator(test_llm_generator,
113
+ prompts,
114
+ sampling_params,
115
+ llm_cb=check_window(prompts))
116
+ check_answers(indices, answer, test_texts)
117
+
118
+
119
+def prep_prompts(batch_size: int):
120
+ """
121
+ Generate prompts which a bunch of assignments,
122
+ then asking for the value of one of them.
123
+ The prompt is just under 10k tokens; sliding window is 4k
124
+ so the answer is outside sliding window, but should still be correct.
125
+ """
126
+ prompts: List[str] = []
127
+ answer: List[int] = []
128
+ indices: List[int] = []
129
+ random.seed(1)
130
+ for _ in range(batch_size):
131
+ idx = random.randint(30, 90)
132
+ indices.append(idx)
133
+ prompt = "```python\n# We set a number of variables, " + \
134
+ f"x{idx} will be important later\n"
135
+ ln = random.randint(800, 1100)
136
+ for k in range(30, ln):
137
+ v = random.randint(10, 99)
138
+ if k == idx:
139
+ answer.append(v)
140
+ prompt += f"x{k} = {v}\n"
141
+ prompt += f"# Now, we check the value of x{idx}:\n"
142
+ prompt += f"assert x{idx} == "
143
+ prompts.append(prompt)
144
+ return prompts, answer, indices
145
+
146
+
147
+def check_answers(indices: List[int], answer: List[int], outputs: List[str]):
148
+ answer2 = [int(text[0:2].strip()) for text in outputs]
149
+ print(list(zip(indices, zip(answer, answer2))))
150
+ numok = 0
151
+ for a1, a2 in zip(answer, answer2):
152
+ if a1 == a2:
153
+ numok += 1
154
+ frac_ok = numok / len(answer)
155
+ print(f"Num OK: {numok}/{len(answer)} {frac_ok}")
156
+ assert frac_ok > 0.7
157
+
158
+
159
+def check_window(prompts: List[str]):
160
+
161
+ def inner(llm: LLM):
162
+ sliding_window = llm.llm_engine.model_config.get_sliding_window()
163
+ assert sliding_window and sliding_window > 0
164
+ assert any(
165
+ len(llm.get_tokenizer().tokenize(prompt)) > sliding_window
166
+ for prompt in prompts)
167
+
168
+ return inner
tests/core/block/test_block_manager_v2.py
+69
@@ -101,3 +101,72 @@ def test_append_slots(block_size, prompt_len, num_slots_to_append,
101
range(prompt_len + num_slots_to_append + num_lookahead_slots)),
102
block_size)) - len(chunk_list(list(range(prompt_len)), block_size))
103
assert num_consumed_blocks == expected_consumed_blocks
104
+
105
+
106
+@pytest.mark.parametrize("block_size", [8, 16])
107
+@pytest.mark.parametrize("prompt_len", [10, 300, 1000])
108
+@pytest.mark.parametrize("num_slots_to_append", [50])
109
+@pytest.mark.parametrize("sliding_window", [20, 32, 200, 512])
110
+def test_sliding_window(block_size, prompt_len, num_slots_to_append,
111
+ sliding_window):
112
+ """Verify append_slots consumes the correct number of blocks from the block
113
+ table.
114
+ """
115
+
116
+ num_gpu_blocks = 1024
117
+ watermark = 0.1
118
+ block_manager = BlockSpaceManagerV2(
119
+ block_size=block_size,
120
+ num_gpu_blocks=num_gpu_blocks,
121
+ num_cpu_blocks=0,
122
+ watermark=watermark,
123
+ sliding_window=sliding_window,
124
+ )
125
+
126
+ def check_used(min_n, max_n=None):
127
+ if max_n is None:
128
+ max_n = min_n
129
+ used = num_gpu_blocks - block_manager.get_num_free_gpu_blocks()
130
+ #print("check", min_n, used, max_n)
131
+ assert min_n <= used
132
+ assert used <= max_n
133
+
134
+ def num_blocks(num_tokens):
135
+ return (num_tokens + block_size - 1) // block_size
136
+
137
+ check_used(0)
138
+
139
+ seq_group = create_seq_group(
140
+ seq_prompt_len=prompt_len,
141
+ seq_output_lens=[0],
142
+ )
143
+
144
+ check_used(0)
145
+
146
+ # Allocate seq
147
+ assert block_manager.can_allocate(seq_group)
148
+ block_manager.allocate(seq_group)
149
+
150
+ check_used(num_blocks(prompt_len))
151
+
152
+ # Seq seq to RUNNING
153
+ seq = seq_group.get_seqs()[0]
154
+ seq.status = SequenceStatus.RUNNING
155
+
156
+ seq.data.update_num_computed_tokens(prompt_len)
157
+ check_used(num_blocks(prompt_len))
158
+
159
+ # this is how we compute it in BlockSpaceManagerV2.__init__
160
+ sliding_blocks = (sliding_window // block_size) + 2
161
+ # plus one block for null block
162
+ sliding_blocks += 1
163
+
164
+ # Append tokens to the sequeqnce
165
+ for token_id in range(num_slots_to_append):
166
+ seq.append_token_id(token_id, {token_id: Logprob(0.0)})
167
+ seq.data.update_num_computed_tokens(1)
168
+ block_manager.append_slots(seq, num_lookahead_slots=0)
169
+ if prompt_len < sliding_window + 10:
170
+ check_used(0, sliding_blocks + 1)
171
+ else:
172
+ check_used(sliding_blocks, sliding_blocks + 1)
vllm/attention/ops/prefix_prefill.py
+5
-1
@@ -697,6 +697,10 @@ if triton.__version__ >= "2.1.0":
697
698
grid = (batch, head, triton.cdiv(max_input_len, BLOCK)) # batch, head,
699
700
+ # 0 means "disable"
701
+ if sliding_window is None or sliding_window <= 0:
702
+ sliding_window = 0
703
+
704
num_warps = 8 if Lk <= 64 else 8
705
if alibi_slopes is not None:
706
_fwd_kernel_alibi[grid](
@@ -794,7 +798,7 @@ if triton.__version__ >= "2.1.0":
798
BLOCK_DMODEL=Lk,
799
BLOCK_DMODEL_PADDED=Lk_padded,
800
BLOCK_N=BLOCK,
797
- SLIDING_WINDOW=sliding_window if sliding_window is not None else 0,
801
+ SLIDING_WINDOW=sliding_window,
802
num_warps=num_warps,
803
num_stages=1,
804
)
vllm/core/block/block_table.py
+32
-2
@@ -20,6 +20,10 @@ class BlockTable:
20
_blocks (Optional[List[Block]], optional): An optional list of existing
21
blocks to initialize the BlockTable with. If not provided, an empty
22
BlockTable is created.
23
+ max_block_sliding_window (Optional[int], optional): The number of
24
+ blocks to keep around for each sequance. If None, all blocks
25
+ are kept (eg., when sliding window is not used).
26
+ It should at least fit the sliding window size of the model.
27
28
Attributes:
29
_block_size (int): The maximum number of tokens that can be stored in a
@@ -37,6 +41,7 @@ class BlockTable:
41
block_size: int,
42
block_allocator: DeviceAwareBlockAllocator,
43
_blocks: Optional[List[Block]] = None,
44
+ max_block_sliding_window: Optional[int] = None,
45
):
46
self._block_size = block_size
47
self._allocator = block_allocator
@@ -44,6 +49,7 @@ class BlockTable:
49
_blocks = []
50
self._blocks: List[Block] = _blocks
51
52
+ self._max_block_sliding_window = max_block_sliding_window
53
# Use helper method instead of directly calculating, as blocks
54
# may not be allocated.
55
self._num_full_slots = len(self._get_all_token_ids())
@@ -89,7 +95,8 @@ class BlockTable:
95
96
def append_token_ids(self,
97
token_ids: List[int],
92
- num_lookahead_slots: int = 0) -> None:
98
+ num_lookahead_slots: int = 0,
99
+ num_computed_slots: Optional[int] = None) -> None:
100
"""Appends a sequence of token IDs to the existing blocks in the
101
BlockTable.
102
@@ -104,13 +111,35 @@ class BlockTable:
111
112
Args:
113
token_ids (List[int]): The sequence of token IDs to be appended.
114
+ num_computed_slots (Optional[int]): The number of KV cache slots
115
+ that are already filled (computed).
116
+ When sliding window is enabled, this is used to compute how many
117
+ blocks to drop at the front of the sequence.
118
+ Without sliding window, None can be passed.
119
+ Without chunked prefill, it should be the same as
120
+ _num_full_slots.
121
"""
108
- assert self._is_allocated
122
+ assert self._is_allocated, "no blocks have been allocated"
123
assert len(self._blocks) > 0
124
125
+ # Drop blocks that are no longer needed due to sliding window
126
+ if self._max_block_sliding_window is not None:
127
+ null_block = self._allocator.allocate_or_get_null_block()
128
+ assert num_computed_slots is not None
129
+ end_block_idx = (num_computed_slots //
130
+ self._block_size) - self._max_block_sliding_window
131
+ for idx in range(0, end_block_idx):
132
+ b = self._blocks[idx]
133
+ if b is not null_block:
134
+ self._allocator.free(b)
135
+ self._blocks[idx] = null_block
136
+
137
+ # Ensure there are enough empty slots for the new tokens plus
138
+ # lookahead slots
139
self.ensure_num_empty_slots(num_empty_slots=len(token_ids) +
140
num_lookahead_slots)
141
142
+ # Update the blocks with the new tokens
143
blocks = self._blocks[self._num_full_slots // self._block_size:]
144
token_blocks = self._chunk_token_blocks_for_append(token_ids)
145
@@ -168,6 +197,7 @@ class BlockTable:
197
block_size=self._block_size,
198
block_allocator=self._allocator,
199
_blocks=forked_blocks,
200
+ max_block_sliding_window=self._max_block_sliding_window,
201
)
202
203
def free(self) -> None:
vllm/core/block/cpu_gpu_block_allocator.py
+74
@@ -105,11 +105,19 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
105
Device.GPU: gpu_block_allocator,
106
}
107
108
+ self._null_block: Optional[Block] = None
109
+
110
self._block_ids_to_allocator: Dict[int, BlockAllocator] = {}
111
for _, allocator in self._allocators.items():
112
for block_id in allocator.all_block_ids:
113
self._block_ids_to_allocator[block_id] = allocator
114
115
+ def allocate_or_get_null_block(self) -> Block:
116
+ if self._null_block is None:
117
+ self._null_block = NullBlock(
118
+ self.allocate_mutable(None, Device.GPU))
119
+ return self._null_block
120
+
121
def allocate_mutable(self, prev_block: Optional[Block],
122
device: Device) -> Block:
123
"""Allocates a new mutable block on the specified device.
@@ -149,6 +157,9 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
157
Args:
158
block (Block): The block to be freed.
159
"""
160
+ # Null block should never be freed
161
+ if isinstance(block, NullBlock):
162
+ return
163
block_id = block.block_id
164
assert block_id is not None
165
allocator = self._block_ids_to_allocator[block_id]
@@ -165,6 +176,8 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
176
List[Block]: A new list of blocks that shares the same memory as the
177
original sequence.
178
"""
179
+ # do not attempt to fork the null block
180
+ assert not isinstance(last_block, NullBlock)
181
block_id = last_block.block_id
182
assert block_id is not None
183
allocator = self._block_ids_to_allocator[block_id]
@@ -226,3 +239,64 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
239
240
def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
241
raise NotImplementedError
242
+
243
+
244
+class NullBlock(Block):
245
+ """
246
+ Null blocks are used as a placeholders for KV cache blocks that have
247
+ been dropped due to sliding window.
248
+ This implementation just wraps an ordinary block and prevents it from
249
+ being modified. It also allows for testing if a block is NullBlock
250
+ via isinstance().
251
+ """
252
+
253
+ def __init__(self, proxy: Block):
254
+ super().__init__()
255
+ self._proxy = proxy
256
+
257
+ def append_token_ids(self, token_ids: List[BlockId]):
258
+ raise ValueError("null block should not be modified")
259
+
260
+ @property
261
+ def block_id(self):
262
+ return self._proxy.block_id
263
+
264
+ @block_id.setter
265
+ def block_id(self, value: Optional[BlockId]):
266
+ raise ValueError("null block should not be modified")
267
+
268
+ @property
269
+ def token_ids(self) -> List[BlockId]:
270
+ return self._proxy.token_ids
271
+
272
+ @property
273
+ def num_empty_slots(self) -> BlockId:
274
+ return self._proxy.num_empty_slots
275
+
276
+ @property
277
+ def is_full(self):
278
+ return self._proxy.is_full
279
+
280
+ @property
281
+ def prev_block(self):
282
+ return self._proxy.prev_block
283
+
284
+ @property
285
+ def computed(self):
286
+ return self._proxy.computed
287
+
288
+ @computed.setter
289
+ def computed(self, value):
290
+ self._proxy.computed = value
291
+
292
+ @property
293
+ def last_accessed(self) -> float:
294
+ return self._proxy.last_accessed
295
+
296
+ @last_accessed.setter
297
+ def last_accessed(self, last_accessed_ts: float):
298
+ self._proxy.last_accessed = last_accessed_ts
299
+
300
+ @property
301
+ def content_hash(self):
302
+ return self._proxy.content_hash
vllm/core/block/interfaces.py
+9
@@ -203,3 +203,12 @@ class DeviceAwareBlockAllocator(ABC):
203
def get_common_computed_block_ids(
204
self, seq_block_ids: List[List[int]]) -> List[int]:
205
pass
206
+
207
+ @abstractmethod
208
+ def allocate_or_get_null_block(self) -> Block:
209
+ """
210
+ Null blocks are used as a placeholders for KV cache blocks that have
211
+ been dropped due to sliding window.
212
+ There is at most one null block per allocator.
213
+ """
214
+ pass
vllm/core/block_manager_v2.py
+17
-7
@@ -66,9 +66,18 @@ class BlockSpaceManagerV2(BlockSpaceManager):
66
self.num_total_gpu_blocks = num_gpu_blocks
67
self.num_total_cpu_blocks = num_cpu_blocks
68
69
- assert sliding_window is None, "Sliding window not yet supported"
70
-
71
- self.block_sliding_window = None
69
+ self.sliding_window = sliding_window
70
+ # max_block_sliding_window is the max number of blocks that need to be
71
+ # allocated
72
+ self.max_block_sliding_window = None
73
+ if sliding_window is not None:
74
+ # +1 here because // rounds down
75
+ num_blocks = sliding_window // block_size + 1
76
+ # +1 here because the last block may not be full,
77
+ # and so the sequence stretches one more block at the beginning
78
+ # For example, if sliding_window is 3 and block_size is 4,
79
+ # we may need 2 blocks when the second block only holds 1 token.
80
+ self.max_block_sliding_window = num_blocks + 1
81
82
self.watermark = watermark
83
assert watermark >= 0.0
@@ -96,10 +105,9 @@ class BlockSpaceManagerV2(BlockSpaceManager):
105
block_size=self.block_size,
106
)
107
99
- assert self.block_sliding_window is None
100
- if self.block_sliding_window is not None:
108
+ if self.max_block_sliding_window is not None:
109
num_required_blocks = min(num_required_blocks,
102
- self.block_sliding_window)
110
+ self.max_block_sliding_window)
111
112
num_free_gpu_blocks = self.block_allocator.get_num_free_blocks(
113
device=Device.GPU)
@@ -125,8 +133,9 @@ class BlockSpaceManagerV2(BlockSpaceManager):
133
block_table = BlockTable(
134
block_size=self.block_size,
135
block_allocator=self.block_allocator,
136
+ max_block_sliding_window=self.max_block_sliding_window,
137
)
129
- assert self.block_sliding_window is None
138
+
139
block_table.allocate(seq.get_token_ids())
140
self.block_tables[seq.seq_id] = block_table
141
@@ -174,6 +183,7 @@ class BlockSpaceManagerV2(BlockSpaceManager):
183
block_table.append_token_ids(
184
token_ids=block_table.get_unseen_token_ids(seq.get_token_ids()),
185
num_lookahead_slots=num_lookahead_slots,
186
+ num_computed_slots=seq.data.get_num_computed_tokens(),
187
)
188
189
# Return any new copy-on-writes.
vllm/engine/arg_utils.py
+2
-1
@@ -648,7 +648,8 @@ class EngineArgs:
648
guided_decoding_backend=self.guided_decoding_backend)
649
650
if (model_config.get_sliding_window() is not None
651
- and scheduler_config.chunked_prefill_enabled):
651
+ and scheduler_config.chunked_prefill_enabled
652
+ and not scheduler_config.use_v2_block_manager):
653
raise ValueError(
654
"Chunked prefill is not supported with sliding window. "
655
"Set --disable-sliding-window to disable sliding window.")
vllm/worker/cache_engine.py
+4
-1
@@ -68,8 +68,11 @@ class CacheEngine:
68
pin_memory = is_pin_memory_available() if device == "cpu" else False
69
kv_cache: List[torch.Tensor] = []
70
for _ in range(self.num_layers):
71
+ # null block in CpuGpuBlockAllocator requires at least that
72
+ # block to be zeroed-out.
73
+ # We zero-out everything for simplicity.
74
kv_cache.append(
72
- torch.empty(kv_cache_shape,
75
+ torch.zeros(kv_cache_shape,
76
dtype=self.dtype,
77
pin_memory=pin_memory,
78
device=device))
vllm/worker/model_runner.py
+49
-24
@@ -269,6 +269,12 @@ class ModelRunner:
269
if len(seq_group_metadata_list) == 0:
270
return ModelInput.empty(self.device)
271
272
+ if self.sliding_window is not None:
273
+ sliding_window_blocks = (self.sliding_window + self.block_size -
274
+ 1) // self.block_size
275
+ block_aligned_sliding_window = \
276
+ sliding_window_blocks * self.block_size
277
+
278
for seq_group_metadata in seq_group_metadata_list:
279
seq_ids = list(seq_group_metadata.seq_data.keys())
280
is_prompt = seq_group_metadata.is_prompt
@@ -309,6 +315,30 @@ class ModelRunner:
315
and self.sliding_window is None
316
and is_prompt)
317
318
+ # These are seq_len/context_len capped to the sliding window.
319
+ # They are passed to decode kernel.
320
+ # We still need original seq_len/context_len to compute slot
321
+ # mapping (and input position) below.
322
+ curr_sliding_window_blocks = None
323
+ sliding_seq_len = seq_len
324
+ sliding_context_len = context_len
325
+
326
+ # TODO(sang): This is a hack to make sliding window work with
327
+ # paged attn. We can remove it if we make paged attn kernel
328
+ # to properly handle slinding window attn.
329
+ if (self.sliding_window is not None and not is_prompt):
330
+ curr_sliding_window_blocks = sliding_window_blocks
331
+ if self.scheduler_config.use_v2_block_manager:
332
+ # number of elements in last block
333
+ suff_len = seq_len % self.block_size
334
+ sliding_seq_len = min(
335
+ seq_len, block_aligned_sliding_window + suff_len)
336
+ if suff_len > 0:
337
+ curr_sliding_window_blocks += 1
338
+ else:
339
+ sliding_seq_len = min(seq_len, self.sliding_window)
340
+ sliding_context_len = sliding_seq_len - 1
341
+
342
# TODO(sang): Combine chunked prefill and prefix caching by
343
# only allowing multiple of block_size chunk size.
344
# NOTE: This only works for oooooooxxx style attention.
@@ -316,6 +346,13 @@ class ModelRunner:
346
assert computed_block_nums is not None
347
context_len = len(computed_block_nums) * self.block_size
348
tokens = tokens[context_len:]
349
+
350
+ # need to think what to set it to when we have both sliding
351
+ # window and prefix caching...
352
+ assert self.sliding_window is None, \
353
+ "Prefix caching is not supported with sliding window"
354
+ sliding_context_len = context_len
355
+
356
if self.attn_backend.get_name() == "flash-attn":
357
# NOTE(woosuk): For flash-attn, the block table should
358
# include the entries for the incoming prefill tokens.
@@ -329,14 +366,9 @@ class ModelRunner:
366
if seq_group_metadata.block_tables is not None:
367
# chunked prefill or decode
368
block_table = seq_group_metadata.block_tables[seq_id]
332
- if self.sliding_window is not None:
333
- # chunked prefill doesn't support sliding window.
334
- assert (not self.scheduler_config.
335
- chunked_prefill_enabled)
336
- sliding_window_blocks = (self.sliding_window //
337
- self.block_size)
338
- block_table = block_table[-sliding_window_blocks:]
339
-
369
+ if curr_sliding_window_blocks is not None:
370
+ block_table = block_table[
371
+ -curr_sliding_window_blocks:]
372
if self.attn_backend.get_name() == "flashinfer":
373
paged_kv_indices.extend(block_table)
374
paged_kv_indptr.append(paged_kv_indptr[-1] +
@@ -354,16 +386,9 @@ class ModelRunner:
386
block_table = []
387
block_tables.append(block_table)
388
357
- # TODO(sang): This is a hack to make sliding window work with
358
- # paged attn. We can remove it if we make paged attn kernel
359
- # to properly handle slinding window attn.
360
- if (self.sliding_window is not None and not is_prompt):
361
- seq_len = min(seq_len, self.sliding_window)
362
- context_len = seq_len - 1
363
-
364
- seq_lens.append(seq_len)
365
- context_lens.append(context_len)
366
- query_len = seq_len - context_len
389
+ seq_lens.append(sliding_seq_len)
390
+ context_lens.append(sliding_context_len)
391
+ query_len = sliding_seq_len - sliding_context_len
392
query_lens.append(query_len)
393
input_tokens.extend(tokens)
394
input_positions.extend(list(range(context_len, seq_len)))
@@ -380,16 +405,15 @@ class ModelRunner:
405
"seq_len: {}, context_len: {}, query_len: {}".format(
406
seq_len, context_len, query_len))
407
num_decode_tokens += query_len
383
- decode_seq_lens.append(seq_len)
408
+ decode_seq_lens.append(sliding_seq_len)
409
410
if lora_id > 0:
411
lora_requests.add(seq_group_metadata.lora_request)
412
388
- lora_index_mapping += [lora_id] * (seq_len - context_len)
413
+ lora_index_mapping += [lora_id] * query_len
414
lora_prompt_mapping.extend(
415
[lora_id] *
391
- (seq_len -
392
- context_len if seq_group_metadata.sampling_params
416
+ (query_len if seq_group_metadata.sampling_params
417
and seq_group_metadata.sampling_params.prompt_logprobs
418
else 1))
419
@@ -417,9 +441,10 @@ class ModelRunner:
441
start_idx = 0
442
if self.sliding_window is not None:
443
if is_prompt:
420
- assert context_len == 0, (
444
+ assert self.scheduler_config.use_v2_block_manager \
445
+ or context_len == 0, (
446
"Prefix caching is currently not supported with "
422
- "sliding window attention")
447
+ "sliding window attention in V1 block manager")
448
# It is an optimization. When it is decoding, it is always
449
# 0. When prefill, we use it to not write slots to kv cache
450
# to save memory.