@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4238bc82f2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Core] Cross-attention KV caching and memory-management (towards eventual encoder/decoder model support) (#4837)

afeldman-nm committed May 29, 2024 at 12:09 UTC 4238bc82f24d5887784b04a353ed93e2360623b4
7 files changed +735 -69
tests/core/block/test_block_manager_v2.py
+153 -1
@@ -1,11 +1,13 @@
1 import pytest
2
3 +from vllm.core.block.utils import (STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE,
4 + STR_NOT_IMPL_ENC_DEC_SWA)
5 from vllm.core.block_manager_v2 import BlockSpaceManagerV2
6 from vllm.core.interfaces import AllocStatus
7 from vllm.sequence import Logprob, SequenceStatus
8 from vllm.utils import chunk_list
9
8 -from ..utils import create_seq_group
10 +from ..utils import create_seq_group, create_seq_group_encoder_decoder
11
12
13 @pytest.mark.parametrize("block_size", [16])
@@ -52,6 +54,156 @@ def test_can_allocate_seq_group(block_size: int, num_seqs_per_group: int,
54 assert can_allocate_result == AllocStatus.LATER
55
56
57 +@pytest.mark.parametrize("block_size", [16])
58 +@pytest.mark.parametrize("num_gpu_blocks", [16, 80, 160])
59 +@pytest.mark.parametrize("num_seqs_per_group", [1, 4])
60 +@pytest.mark.parametrize("watermark", [0.0, 0.5])
61 +def test_can_allocate_seq_group_encoder_decoder(block_size: int,
62 + num_seqs_per_group: int,
63 + num_gpu_blocks: int,
64 + watermark: float):
65 + block_manager = BlockSpaceManagerV2(
66 + block_size=block_size,
67 + num_gpu_blocks=num_gpu_blocks,
68 + num_cpu_blocks=1024,
69 + watermark=watermark,
70 + )
71 + num_watermark_blocks = int(watermark * num_gpu_blocks)
72 +
73 + num_output_blocks_per_seq = 1
74 +
75 + # NOTE: This should be num_output_blocks_per_seq * num_seqs_per_group, but
76 + # the current implementation assumes all seqs are new prompts / don't have
77 + # different output lens.
78 + num_output_blocks = num_output_blocks_per_seq
79 +
80 + for bdx, num_prompt_blocks in enumerate(
81 + range(1, num_gpu_blocks - num_output_blocks)):
82 + num_cross_blocks_per_seq = num_prompt_blocks
83 +
84 + seq_group = create_seq_group_encoder_decoder(
85 + seq_prompt_len=block_size * num_prompt_blocks,
86 + seq_output_lens=[
87 + block_size * num_output_blocks_per_seq
88 + for _ in range(num_seqs_per_group)
89 + ],
90 + request_id=str(bdx))
91 +
92 + assert num_prompt_blocks + num_output_blocks <= num_gpu_blocks
93 +
94 + can_allocate_result = block_manager.can_allocate(seq_group)
95 +
96 + num_required_blocks = num_prompt_blocks + \
97 + num_output_blocks + \
98 + num_cross_blocks_per_seq
99 +
100 + if num_gpu_blocks - num_required_blocks < num_watermark_blocks:
101 + assert can_allocate_result == AllocStatus.NEVER
102 + elif num_gpu_blocks >= num_required_blocks:
103 + assert can_allocate_result == AllocStatus.OK
104 + else:
105 + assert can_allocate_result == AllocStatus.LATER
106 +
107 +
108 +@pytest.mark.parametrize("block_size", [16])
109 +@pytest.mark.parametrize("num_gpu_blocks", [16])
110 +@pytest.mark.parametrize("num_seqs_per_group", [1])
111 +@pytest.mark.parametrize("watermark", [0.0, 0.5])
112 +def test_can_allocate_encoder_decoder_fails_with_swa(block_size: int,
113 + num_seqs_per_group: int,
114 + num_gpu_blocks: int,
115 + watermark: float):
116 + '''
117 + SWA short for Sliding Window Attention.
118 +
119 + At time of writing block manager v2 does not support SWA.
120 +
121 + However even when SWA is implemented for block manager v2,
122 + there will still most likely be a separate workstream required
123 + to enable SWA for encoder/decoder models.
124 +
125 + Therefore this test enforces that one of the following cases
126 + hold true:
127 + 1. Block manager v2 does not support SWA at all (true at time of writing)
128 + 2. Block manager v2 fails with NotImplementError when SWA is enabled
129 + AND a SequenceGroup with an encoder sequence (i.e. in support of an
130 + encoder/decoder model) is passed into can_allocate() as an argument
131 +
132 + The setup for this test is stripped down version of
133 + test_can_allocate_seq_group_encoder_decoder()
134 + '''
135 +
136 + with pytest.raises((NotImplementedError, AssertionError)) as exc_info:
137 + block_manager = BlockSpaceManagerV2(
138 + block_size=block_size,
139 + num_gpu_blocks=num_gpu_blocks,
140 + num_cpu_blocks=1024,
141 + watermark=watermark,
142 + sliding_window=5 # SWA
143 + )
144 +
145 + num_output_blocks_per_seq = 1
146 + num_prompt_blocks = 1
147 + num_output_blocks = num_output_blocks_per_seq
148 + seq_group = create_seq_group_encoder_decoder(
149 + seq_prompt_len=block_size * num_prompt_blocks,
150 + seq_output_lens=[
151 + block_size * num_output_blocks_per_seq
152 + for _ in range(num_seqs_per_group)
153 + ],
154 + request_id="0")
155 +
156 + assert num_prompt_blocks + num_output_blocks <= num_gpu_blocks
157 + block_manager.can_allocate(seq_group)
158 +
159 + # Assert that either
160 + # 1. Block manager v2 constructor fails with assertion that sliding window
161 + # is not yet supported (most likely near-term outcome at time of
162 + # writing), or
163 + # 2. can_allocate() fails with NotImplementedError due to combination of
164 + # encoder/decoder and sliding window attention
165 + if isinstance(exc_info.value, NotImplementedError):
166 + assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_SWA
167 + elif isinstance(exc_info.value, AssertionError):
168 + assert str(exc_info.value) == "Sliding window not yet supported"
169 +
170 +
171 +@pytest.mark.parametrize("block_size", [16])
172 +@pytest.mark.parametrize("num_gpu_blocks", [16])
173 +@pytest.mark.parametrize("num_seqs_per_group", [1])
174 +@pytest.mark.parametrize("watermark", [0.0, 0.5])
175 +def test_can_allocate_encoder_decoder_fails_with_prefix_cache(
176 + block_size: int, num_seqs_per_group: int, num_gpu_blocks: int,
177 + watermark: float):
178 +
179 + block_manager = BlockSpaceManagerV2(
180 + block_size=block_size,
181 + num_gpu_blocks=num_gpu_blocks,
182 + num_cpu_blocks=1024,
183 + watermark=watermark,
184 + enable_caching=True # Prefix cache
185 + )
186 +
187 + num_output_blocks_per_seq = 1
188 + num_prompt_blocks = 1
189 + num_output_blocks = num_output_blocks_per_seq
190 + seq_group = create_seq_group_encoder_decoder(
191 + seq_prompt_len=block_size * num_prompt_blocks,
192 + seq_output_lens=[
193 + block_size * num_output_blocks_per_seq
194 + for _ in range(num_seqs_per_group)
195 + ],
196 + request_id="0")
197 +
198 + assert num_prompt_blocks + num_output_blocks <= num_gpu_blocks
199 +
200 + # Assert that either can_allocate() fails with NotImplementedError
201 + # due to combination of encoder/decoder and prefix cache
202 + with pytest.raises(NotImplementedError) as exc_info:
203 + block_manager.can_allocate(seq_group)
204 + assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE
205 +
206 +
207 @pytest.mark.parametrize("block_size", [1, 8])
208 @pytest.mark.parametrize("prompt_len", [1, 7, 8])
209 @pytest.mark.parametrize("num_slots_to_append", [1, 8, 129])
tests/core/test_block_manager.py
+217 -3
@@ -6,13 +6,15 @@ import pytest
6
7 from vllm import SamplingParams
8 from vllm.block import PhysicalTokenBlock
9 +from vllm.core.block.utils import (STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE,
10 + STR_NOT_IMPL_ENC_DEC_SWA)
11 from vllm.core.block_manager_v1 import (BlockSpaceManagerV1,
12 UncachedBlockAllocator)
13 from vllm.core.interfaces import AllocStatus
14 from vllm.sequence import Logprob, Sequence, SequenceGroup, SequenceStatus
15 from vllm.utils import Device
16
15 -from .utils import create_dummy_prompt
17 +from .utils import create_dummy_prompt, create_dummy_prompt_encoder_decoder
18
19
20 def test_block_allocator_allocate():
@@ -73,7 +75,7 @@ def test_allocate():
75 # Allocate same sequence group to all available gpu blocks.
76 for i in range(num_gpu_blocks):
77 _, seq_group = create_dummy_prompt(str(i), block_size)
76 - assert block_manager.can_allocate(seq_group)
78 + assert block_manager.can_allocate(seq_group) == AllocStatus.OK
79 block_manager.allocate(seq_group)
80 assert block_manager.can_allocate(seq_group) != AllocStatus.OK
81
@@ -85,11 +87,107 @@ def test_allocate():
87 watermark=1 / num_gpu_blocks)
88 for i in range(num_gpu_blocks - 1):
89 _, seq_group = create_dummy_prompt(str(i), block_size)
88 - assert block_manager.can_allocate(seq_group)
90 + assert block_manager.can_allocate(seq_group) == AllocStatus.OK
91 block_manager.allocate(seq_group)
92 assert block_manager.can_allocate(seq_group) != AllocStatus.OK
93
94
95 +def test_allocate_encoder_decoder():
96 + block_size = 4
97 + num_cpu_blocks = 4
98 + num_gpu_blocks = 4
99 + block_req_per_seq_group = 2
100 + block_manager = BlockSpaceManagerV1(block_size,
101 + num_cpu_blocks,
102 + num_gpu_blocks,
103 + watermark=0)
104 +
105 + # Allocate same sequence group to all available gpu blocks.
106 + for i in range(num_gpu_blocks // block_req_per_seq_group):
107 + _, _, seq_group = create_dummy_prompt_encoder_decoder(
108 + str(i),
109 + decoder_prompt_length=block_size,
110 + encoder_prompt_length=block_size)
111 + assert block_manager.can_allocate(seq_group) == AllocStatus.OK
112 + block_manager.allocate(seq_group)
113 + assert block_manager.can_allocate(seq_group) != AllocStatus.OK
114 +
115 + # Allocate same sequence group to all available gpu blocks.
116 + # Use watermark to reserve one gpu block.
117 + block_manager = BlockSpaceManagerV1(block_size,
118 + num_cpu_blocks,
119 + num_gpu_blocks,
120 + watermark=1 / num_gpu_blocks)
121 + for i in range((num_gpu_blocks - 1) // block_req_per_seq_group):
122 + _, _, seq_group = create_dummy_prompt_encoder_decoder(
123 + str(i),
124 + decoder_prompt_length=block_size,
125 + encoder_prompt_length=block_size)
126 + assert block_manager.can_allocate(seq_group) == AllocStatus.OK
127 + block_manager.allocate(seq_group)
128 + assert block_manager.can_allocate(seq_group) != AllocStatus.OK
129 +
130 +
131 +def test_allocate_encoder_decoder_fails_with_swa():
132 + # SWA short for sliding window attention
133 +
134 + block_size = 4
135 + num_cpu_blocks = 4
136 + num_gpu_blocks = 4
137 + block_manager = BlockSpaceManagerV1(block_size,
138 + num_cpu_blocks,
139 + num_gpu_blocks,
140 + watermark=0,
141 + sliding_window=5) # swa
142 +
143 + # Allocate same sequence group to all available gpu blocks.
144 + _, _, seq_group = create_dummy_prompt_encoder_decoder(
145 + "0",
146 + decoder_prompt_length=block_size,
147 + encoder_prompt_length=block_size)
148 +
149 + # Assert that can_allocate() fails due to SWA
150 + with pytest.raises(NotImplementedError) as exc_info:
151 + block_manager.can_allocate(seq_group)
152 +
153 + assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_SWA
154 +
155 + # Assert that allocate() fails due to SWA
156 + with pytest.raises(NotImplementedError) as exc_info:
157 + block_manager.allocate(seq_group)
158 +
159 + assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_SWA
160 +
161 +
162 +def test_allocate_encoder_decoder_fails_with_prefix_caching():
163 + block_size = 4
164 + num_cpu_blocks = 4
165 + num_gpu_blocks = 4
166 + block_manager = BlockSpaceManagerV1(block_size,
167 + num_cpu_blocks,
168 + num_gpu_blocks,
169 + watermark=0,
170 + enable_caching=True) # Prefix cache
171 +
172 + # Allocate same sequence group to all available gpu blocks.
173 + _, _, seq_group = create_dummy_prompt_encoder_decoder(
174 + "0",
175 + decoder_prompt_length=block_size,
176 + encoder_prompt_length=block_size)
177 +
178 + # Assert that can_allocate() fails due to prefix caching
179 + with pytest.raises(NotImplementedError) as exc_info:
180 + block_manager.can_allocate(seq_group)
181 +
182 + assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE
183 +
184 + # Assert that allocate() fails due to prefix caching
185 + with pytest.raises(NotImplementedError) as exc_info:
186 + block_manager.allocate(seq_group)
187 +
188 + assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE
189 +
190 +
191 def test_append_slot_single_seq():
192 block_size = 4
193 num_cpu_blocks = 4
@@ -244,6 +342,62 @@ def test_swap():
342 assert before_gpu_blocks == after_gpu_blocks + len(cpu_blocks)
343
344
345 +def test_swap_encoder_decoder():
346 + block_size = 4
347 + num_cpu_blocks = 4
348 + num_gpu_blocks = 4
349 + block_manager = BlockSpaceManagerV1(block_size,
350 + num_cpu_blocks,
351 + num_gpu_blocks,
352 + watermark=0)
353 +
354 + decoder_prompt, encoder_prompt, seq_group = \
355 + create_dummy_prompt_encoder_decoder(
356 + "1",
357 + decoder_prompt_length=block_size,
358 + encoder_prompt_length=block_size)
359 + decoder_prompt.status = SequenceStatus.WAITING
360 + encoder_prompt.status = SequenceStatus.WAITING
361 + block_manager.allocate(seq_group)
362 +
363 + # Emulate a forward pass by appending a single token.
364 + # The block manager then knows how many unprocessed
365 + # tokens will be written in the next forward pass.
366 + token_id = 0
367 + decoder_prompt.status = SequenceStatus.RUNNING
368 + decoder_prompt.append_token_id(token_id, {token_id: Logprob(0.0)})
369 +
370 + # Swap encoder/decoder seq group from GPU -> CPU.
371 + decoder_gpu_blocks = block_manager.get_block_table(decoder_prompt)
372 + cross_gpu_blocks = block_manager.get_cross_block_table(seq_group)
373 + gpu_blocks = decoder_gpu_blocks + cross_gpu_blocks
374 + assert block_manager.can_swap_out(seq_group)
375 + before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
376 + before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
377 + mapping = block_manager.swap_out(seq_group)
378 + assert [x[0] for x in mapping] == gpu_blocks
379 + #assert list(mapping.keys()) == gpu_blocks
380 + after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
381 + after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
382 + assert before_cpu_blocks == after_cpu_blocks + len(gpu_blocks)
383 + assert before_gpu_blocks + len(gpu_blocks) == after_gpu_blocks
384 + decoder_prompt.status = SequenceStatus.SWAPPED
385 +
386 + # Swap encoder/decoder seq group from CPU -> GPU.
387 + decoder_cpu_blocks = block_manager.get_block_table(decoder_prompt)
388 + cross_cpu_blocks = block_manager.get_cross_block_table(seq_group)
389 + cpu_blocks = decoder_cpu_blocks + cross_cpu_blocks
390 + assert block_manager.can_swap_in(seq_group) == AllocStatus.OK
391 + before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
392 + before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
393 + mapping = block_manager.swap_in(seq_group)
394 + assert [x[0] for x in mapping] == cpu_blocks
395 + after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
396 + after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
397 + assert before_cpu_blocks + len(cpu_blocks) == after_cpu_blocks
398 + assert before_gpu_blocks == after_gpu_blocks + len(cpu_blocks)
399 +
400 +
401 def test_free():
402 block_size = 4
403 num_cpu_blocks = 4
@@ -268,6 +422,41 @@ def test_free():
422 block_manager.get_block_table(prompt)
423
424
425 +def test_free_encoder_decoder():
426 + block_size = 4
427 + num_cpu_blocks = 4
428 + num_gpu_blocks = 4
429 + block_manager = BlockSpaceManagerV1(block_size,
430 + num_cpu_blocks,
431 + num_gpu_blocks,
432 + watermark=0)
433 +
434 + decoder_prompt, encoder_prompt, seq_group = \
435 + create_dummy_prompt_encoder_decoder(
436 + "1",
437 + decoder_prompt_length=block_size,
438 + encoder_prompt_length=block_size)
439 + block_manager.allocate(seq_group)
440 +
441 + # Free allocated seq.
442 + decoder_prompt_blocks = len(block_manager.get_block_table(decoder_prompt))
443 + encoder_prompt_blocks = len(block_manager.get_cross_block_table(seq_group))
444 + prompt_blocks = decoder_prompt_blocks + encoder_prompt_blocks
445 + before_blocks = block_manager.get_num_free_gpu_blocks()
446 + block_manager.free(decoder_prompt)
447 + block_manager.free_cross(seq_group)
448 + after_blocks = block_manager.get_num_free_gpu_blocks()
449 + assert after_blocks == before_blocks + prompt_blocks
450 +
451 + # Block table for freed encoder & decoder seq's are deleted.
452 + with pytest.raises(KeyError):
453 + block_manager.get_block_table(decoder_prompt)
454 +
455 + # Block table for freed encoder & decoder seq's are deleted.
456 + with pytest.raises(KeyError):
457 + block_manager.get_block_table(encoder_prompt)
458 +
459 +
460 def test_reset():
461 block_size = 4
462 num_cpu_blocks = 4
@@ -289,6 +478,31 @@ def test_reset():
478 assert block_manager.get_num_free_gpu_blocks() == original_blocks
479
480
481 +def test_reset_encoder_decoder():
482 + block_size = 4
483 + num_cpu_blocks = 4
484 + num_gpu_blocks = 4
485 + block_req_per_seq_group = 2
486 + block_manager = BlockSpaceManagerV1(block_size,
487 + num_cpu_blocks,
488 + num_gpu_blocks,
489 + watermark=0)
490 +
491 + # Allocate same seq group on all available gpu blocks.
492 + original_blocks = block_manager.get_num_free_gpu_blocks()
493 + for i in range(num_gpu_blocks // block_req_per_seq_group):
494 + _, _, seq_group = create_dummy_prompt_encoder_decoder(
495 + f"{i}",
496 + decoder_prompt_length=block_size,
497 + encoder_prompt_length=block_size)
498 + block_manager.allocate(seq_group)
499 + assert block_manager.get_num_free_gpu_blocks() == 0
500 +
501 + # Resetting block manager frees all allocated blocks.
502 + block_manager.reset()
503 + assert block_manager.get_num_free_gpu_blocks() == original_blocks
504 +
505 +
506 def test_sliding_window_multi_seq():
507 """
508 Tests that memory allocation and deallocation is handled
tests/core/utils.py
+98 -1
@@ -39,6 +39,52 @@ def create_dummy_prompt(
39 return prompt, seq_group
40
41
42 +def create_dummy_prompt_encoder_decoder(
43 + request_id: str,
44 + decoder_prompt_length: int,
45 + encoder_prompt_length: int,
46 + block_size: Optional[int] = None,
47 + lora_request: Optional[LoRARequest] = None,
48 + use_beam_search: bool = False,
49 + best_of: int = 1,
50 +) -> Tuple[Sequence, SequenceGroup]:
51 + if not block_size:
52 + block_size = decoder_prompt_length
53 +
54 + # Create dummy prompt sequence with tokens 0...block_size-1
55 + # and prompt "0 ... block_size".
56 + decoder_prompt_tokens = list(range(decoder_prompt_length))
57 + decoder_prompt_str = " ".join([str(t) for t in decoder_prompt_tokens])
58 +
59 + decoder_prompt = Sequence(int(request_id),
60 + inputs={
61 + "prompt": decoder_prompt_str,
62 + "prompt_token_ids": decoder_prompt_tokens,
63 + "multi_modal_data": None,
64 + },
65 + block_size=block_size)
66 +
67 + encoder_prompt_tokens = list(reversed(list(range(encoder_prompt_length))))
68 + encoder_prompt_str = " ".join([str(t) for t in encoder_prompt_tokens])
69 + encoder_prompt = Sequence(int(request_id),
70 + inputs={
71 + "prompt": encoder_prompt_str,
72 + "prompt_token_ids": encoder_prompt_tokens,
73 + "multi_modal_data": None,
74 + },
75 + block_size=block_size)
76 + seq_group = SequenceGroup(request_id=request_id,
77 + seqs=[decoder_prompt],
78 + sampling_params=SamplingParams(
79 + use_beam_search=use_beam_search,
80 + best_of=best_of),
81 + arrival_time=time.time(),
82 + lora_request=lora_request,
83 + encoder_seq=encoder_prompt)
84 +
85 + return decoder_prompt, encoder_prompt, seq_group
86 +
87 +
88 def create_seq_group(
89 seq_prompt_len: int = 1024,
90 seq_output_lens: Iterable[int] = (128, ),
@@ -82,5 +128,56 @@ def create_seq_group(
128 return seq_group
129
130
131 +def create_seq_group_encoder_decoder(
132 + seq_prompt_len: int = 1024,
133 + seq_output_lens: Iterable[int] = (128, ),
134 + request_id: str = '0',
135 + seq_id_start: int = 0,
136 + sampling_params: Optional[SamplingParams] = None) -> SequenceGroup:
137 +
138 + assert len(seq_output_lens) > 0
139 +
140 + if sampling_params is None:
141 + sampling_params = SamplingParams()
142 +
143 + prompt_token_ids = [0] * seq_prompt_len
144 +
145 + seqs = []
146 + for seq_id_offset, output_len in enumerate(seq_output_lens):
147 + seq = Sequence(
148 + seq_id=seq_id_start + seq_id_offset,
149 + inputs={
150 + "prompt": "",
151 + "prompt_token_ids": prompt_token_ids,
152 + "multi_modal_data": None,
153 + },
154 + block_size=16,
155 + )
156 +
157 + for i in range(output_len):
158 + seq.append_token_id(
159 + token_id=i,
160 + logprobs={i: Logprob(0.0)},
161 + )
162 + seqs.append(seq)
163 +
164 + # Encoder sequence
165 + encoder_seq = Sequence(
166 + seq_id=seq_id_start + len(seq_output_lens),
167 + inputs={
168 + "prompt": "",
169 + "prompt_token_ids": prompt_token_ids,
170 + "multi_modal_data": None,
171 + },
172 + block_size=16,
173 + )
174 +
175 + return SequenceGroup(request_id=request_id,
176 + seqs=seqs,
177 + sampling_params=sampling_params,
178 + arrival_time=time.time(),
179 + encoder_seq=encoder_seq)
180 +
181 +
182 def round_up_to_next_block(seq_len: int, block_size: int) -> int:
86 - return (seq_len + block_size - 1) // block_size
183 + return (seq_len + block_size - 1) // block_size
\ No newline at end of file
vllm/core/block/utils.py new
+56
@@ -0,0 +1,56 @@
1 +"""Block manager utils."""
2 +from vllm.sequence import SequenceGroup
3 +
4 +# Exception strings for non-implemented block manager enc/dec scenarios
5 +
6 +STR_NOT_IMPL_ENC_DEC_SWA = \
7 + "Sliding window attention for encoder/decoder models " + \
8 + "is not currently supported."
9 +
10 +STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE = \
11 + "Prefix caching for encoder/decoder models " + \
12 + "is not currently supported."
13 +
14 +
15 +def _get_block_mgr_sliding_window_attr(block_mgr):
16 + '''
17 + BlockManagerV1 and BlockManagerV2 have slightly different
18 + members related to sliding window attention (SWA). This
19 + function extracts the appropriate member to use for determining
20 + whether SWA is enabled.
21 +
22 + Arguments:
23 +
24 + * block_mgr: BlockManagerV1 or BlockManagerV2 instance
25 + '''
26 +
27 + if hasattr(block_mgr, 'block_sliding_window'):
28 + return block_mgr.block_sliding_window
29 + if hasattr(block_mgr, 'max_block_sliding_window'):
30 + return block_mgr.max_block_sliding_window
31 +
32 + raise AttributeError("Block manager instance has neither " + \
33 + "block_sliding_window nor " + \
34 + "max_block_sliding_window attributes.")
35 +
36 +
37 +def check_no_caching_or_swa_for_blockmgr_encdec(
38 + block_mgr, seq_group: SequenceGroup) -> None:
39 + '''
40 + Enforce that prefix caching & sliding-window attention (SWA)
41 + are currently unsupported *specifically* for encoder/decoder models.
42 +
43 + Raises NotImplementedError if unsupported scenario is detected.
44 +
45 + Arguments:
46 +
47 + * block_mgr: BlockSpaceManager instance
48 + * seq_group: SequenceGroup passed to block_mgr
49 + '''
50 +
51 + if seq_group.is_encoder_decoder():
52 + if _get_block_mgr_sliding_window_attr(block_mgr) is not None:
53 + raise NotImplementedError(STR_NOT_IMPL_ENC_DEC_SWA)
54 +
55 + if block_mgr.enable_caching:
56 + raise NotImplementedError(STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE)
vllm/core/block_manager_v1.py
+132 -55
@@ -8,6 +8,7 @@ from typing import Sequence as GenericSequence
8 from typing import Set, Tuple
9
10 from vllm.block import BlockTable, PhysicalTokenBlock
11 +from vllm.core.block.utils import check_no_caching_or_swa_for_blockmgr_encdec
12 from vllm.core.evictor_v1 import EvictionPolicy, Evictor, make_evictor
13 from vllm.core.interfaces import AllocStatus, BlockSpaceManager
14 from vllm.logger import init_logger
@@ -255,14 +256,30 @@ class BlockSpaceManagerV1(BlockSpaceManager):
256 Device.CPU, block_size, num_cpu_blocks)
257 # Mapping: seq_id -> BlockTable.
258 self.block_tables: Dict[int, BlockTable] = {}
259 + # Mapping: req_id -> BlockTable
260 + # Note that each SequenceGroup has a unique
261 + # request ID
262 + self.cross_block_tables: Dict[str, BlockTable] = {}
263 +
264 + def _get_seq_num_required_blocks(self, seq: Sequence) -> int:
265 + return 0 if seq is None \
266 + else len(seq.logical_token_blocks)
267
268 def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
269 # FIXME(woosuk): Here we assume that all sequences in the group share
270 # the same prompt. This may not be true for preempted sequences.
262 - seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
263 - num_required_blocks = len(seq.logical_token_blocks)
271 +
272 + check_no_caching_or_swa_for_blockmgr_encdec(self, seq_group)
273 +
274 + self_num_required_blocks = self._get_seq_num_required_blocks(
275 + seq_group.get_seqs(status=SequenceStatus.WAITING)[0])
276 + cross_num_required_blocks = self._get_seq_num_required_blocks(
277 + seq_group.get_encoder_seq())
278 + num_required_blocks = self_num_required_blocks + \
279 + cross_num_required_blocks
280
281 if self.block_sliding_window is not None:
282 +
283 num_required_blocks = min(num_required_blocks,
284 self.block_sliding_window)
285 num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
@@ -276,11 +293,10 @@ class BlockSpaceManagerV1(BlockSpaceManager):
293 else:
294 return AllocStatus.LATER
295
279 - def allocate(self, seq_group: SequenceGroup) -> None:
280 - # NOTE: Here we assume that all sequences in the group have the same
281 - # prompt.
282 - seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
283 -
296 + def _allocate_sequence(self, \
297 + seq: Sequence, \
298 + ref_count: int, \
299 + is_encoder_decoder: bool = True) -> BlockTable:
300 # Allocate new physical token blocks that will store the prompt tokens.
301 num_prompt_blocks = len(seq.logical_token_blocks)
302
@@ -290,21 +306,46 @@ class BlockSpaceManagerV1(BlockSpaceManager):
306 and logical_idx >= self.block_sliding_window):
307 block = block_table[logical_idx % self.block_sliding_window]
308 # Set the reference counts of the token blocks.
293 - block.ref_count = seq_group.num_seqs()
294 - elif self.enable_caching:
309 + block.ref_count = ref_count
310 + elif not is_encoder_decoder and self.enable_caching:
311 block = self.gpu_allocator.allocate(
312 seq.hash_of_block(logical_idx),
313 seq.num_hashed_tokens_of_block(logical_idx))
314 else:
315 block = self.gpu_allocator.allocate()
316 # Set the reference counts of the token blocks.
301 - block.ref_count = seq_group.num_seqs()
317 + block.ref_count = ref_count
318 block_table.append(block)
319
304 - # Assign the block table for each sequence.
320 + return block_table
321 +
322 + def allocate(self, seq_group: SequenceGroup) -> None:
323 + is_encoder_decoder = seq_group.is_encoder_decoder()
324 + check_no_caching_or_swa_for_blockmgr_encdec(self, seq_group)
325 +
326 + # Allocate decoder sequences
327 + #
328 + # NOTE: Here we assume that all sequences in the group have the same
329 + # decoder prompt.
330 + seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
331 + block_table: BlockTable = \
332 + self._allocate_sequence(seq,
333 + seq_group.num_seqs(),
334 + is_encoder_decoder)
335 +
336 + # Assign the self-attention block tables for each sequence.
337 for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
338 self.block_tables[seq.seq_id] = block_table.copy()
339
340 + # Allocate encoder sequence
341 + if is_encoder_decoder:
342 + # A SequenceGroup has only a single encoder sequence (at most),
343 + # thus allocate with a ref count of 1
344 + block_table = self._allocate_sequence(seq_group.get_encoder_seq(),
345 + 1, is_encoder_decoder)
346 + # Assign the cross-attention block table for the SequenceGroup.
347 + self.cross_block_tables[seq_group.request_id] = block_table
348 +
349 def can_append_slots(self,
350 seq_group: SequenceGroup,
351 num_lookahead_slots: int = 0) -> bool:
@@ -443,13 +484,18 @@ class BlockSpaceManagerV1(BlockSpaceManager):
484
485 def _get_physical_blocks(
486 self, seq_group: SequenceGroup) -> List[PhysicalTokenBlock]:
487 +
488 # NOTE: Here, we assume that the physical blocks are only shared by
489 # the sequences in the same group.
490 + request_id = seq_group.request_id
491 blocks: Set[PhysicalTokenBlock] = set()
492 for seq in seq_group.get_seqs():
493 if seq.is_finished():
494 continue
495 blocks.update(self.block_tables[seq.seq_id])
496 + # Cross-attention blocks
497 + if seq_group.is_encoder_decoder():
498 + blocks.update(self.cross_block_tables[request_id])
499 return list(blocks)
500
501 def can_swap_in(self,
@@ -457,8 +503,11 @@ class BlockSpaceManagerV1(BlockSpaceManager):
503 num_lookahead_slots: int = 0) -> AllocStatus:
504 assert (num_lookahead_slots == 0
505 ), "BlockSpaceManagerV1 does not support lookahead allocation"
506 +
507 blocks = self._get_physical_blocks(seq_group)
508 num_swapped_seqs = seq_group.num_seqs(status=SequenceStatus.SWAPPED)
509 + if seq_group.is_encoder_decoder():
510 + num_swapped_seqs += 1
511 num_free_blocks = self.gpu_allocator.get_num_free_blocks()
512 # NOTE: Conservatively, we assume that every sequence will allocate
513 # at least one free block right after the swap-in.
@@ -471,70 +520,81 @@ class BlockSpaceManagerV1(BlockSpaceManager):
520 else:
521 return AllocStatus.LATER
522
523 + def _swap_block_table(
524 + self, block_table: BlockTable, src_allocator: BlockAllocatorBase,
525 + dest_allocator: BlockAllocatorBase,
526 + mapping: Dict[PhysicalTokenBlock,
527 + PhysicalTokenBlock]) -> BlockTable:
528 + new_block_table = []
529 +
530 + for from_block in block_table:
531 + if from_block in mapping:
532 + to_block = mapping[from_block]
533 + to_block.ref_count += 1
534 + else:
535 + to_block = dest_allocator.allocate(
536 + from_block.block_hash, from_block.num_hashed_tokens)
537 + mapping[from_block] = to_block
538 + new_block_table.append(to_block)
539 + # Free the source block swapped in to destination.
540 + src_allocator.free(from_block)
541 +
542 + return new_block_table
543 +
544 def swap_in(self,
545 seq_group: SequenceGroup,
546 num_lookahead_slots: int = 0) -> List[Tuple[int, int]]:
547 assert (num_lookahead_slots == 0
548 ), "BlockSpaceManagerV1 does not support lookahead allocation"
549
550 + request_id = seq_group.request_id
551 +
552 # CPU block -> GPU block.
553 # dict is efficient in lookup `if cpu_block in mapping`
554 mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
555 for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
484 - new_block_table: BlockTable = []
485 - block_table = self.block_tables[seq.seq_id]
486 -
487 - for cpu_block in block_table:
488 - if cpu_block in mapping:
489 - gpu_block = mapping[cpu_block]
490 - gpu_block.ref_count += 1
491 - else:
492 - gpu_block = self.gpu_allocator.allocate(
493 - cpu_block.block_hash, cpu_block.num_hashed_tokens)
494 - mapping[cpu_block] = gpu_block
495 - new_block_table.append(gpu_block)
496 - # Free the CPU block swapped in to GPU.
497 - self.cpu_allocator.free(cpu_block)
498 - self.block_tables[seq.seq_id] = new_block_table
499 -
500 - block_number_mapping = {
501 - cpu_block.block_number: gpu_block.block_number
502 - for cpu_block, gpu_block in mapping.items()
503 - }
504 - # convert to list of tuples once here
505 - return list(block_number_mapping.items())
556 + self.block_tables[seq.seq_id] = \
557 + self._swap_block_table(self.block_tables[seq.seq_id],
558 + self.cpu_allocator,
559 + self.gpu_allocator,
560 + mapping)
561 +
562 + if seq_group.is_encoder_decoder():
563 + self.cross_block_tables[request_id] = \
564 + self._swap_block_table(self.cross_block_tables[request_id],
565 + self.cpu_allocator,
566 + self.gpu_allocator,
567 + mapping)
568 +
569 + return [(cpu_block.block_number, gpu_block.block_number)
570 + for cpu_block, gpu_block in mapping.items()]
571
572 def can_swap_out(self, seq_group: SequenceGroup) -> bool:
573 blocks = self._get_physical_blocks(seq_group)
574 return len(blocks) <= self.cpu_allocator.get_num_free_blocks()
575
576 def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
577 + request_id = seq_group.request_id
578 +
579 # GPU block -> CPU block.
580 # dict is efficient in lookup `if gpu_block in mapping`
581 mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
582 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
516 - new_block_table: BlockTable = []
517 - block_table = self.block_tables[seq.seq_id]
518 -
519 - for gpu_block in block_table:
520 - if gpu_block in mapping:
521 - cpu_block = mapping[gpu_block]
522 - cpu_block.ref_count += 1
523 - else:
524 - cpu_block = self.cpu_allocator.allocate(
525 - gpu_block.block_hash, gpu_block.num_hashed_tokens)
526 - mapping[gpu_block] = cpu_block
527 - new_block_table.append(cpu_block)
528 - # Free the GPU block swapped out to CPU.
529 - self.gpu_allocator.free(gpu_block)
530 - self.block_tables[seq.seq_id] = new_block_table
531 -
532 - block_number_mapping = {
533 - gpu_block.block_number: cpu_block.block_number
534 - for gpu_block, cpu_block in mapping.items()
535 - }
536 - # convert to list of tuples once here
537 - return list(block_number_mapping.items())
583 + self.block_tables[seq.seq_id] = \
584 + self._swap_block_table(self.block_tables[seq.seq_id],
585 + self.gpu_allocator,
586 + self.cpu_allocator,
587 + mapping)
588 +
589 + if seq_group.is_encoder_decoder():
590 + self.cross_block_tables[request_id] = \
591 + self._swap_block_table(self.cross_block_tables[request_id],
592 + self.gpu_allocator,
593 + self.cpu_allocator,
594 + mapping)
595 +
596 + return [(cpu_block.block_number, gpu_block.block_number)
597 + for cpu_block, gpu_block in mapping.items()]
598
599 def _free_block_table(self, block_table: BlockTable) -> None:
600 # when using a sliding window, each seq will only use up
@@ -559,15 +619,32 @@ class BlockSpaceManagerV1(BlockSpaceManager):
619 self._free_block_table(block_table)
620 del self.block_tables[seq.seq_id]
621
622 + def free_cross(self, seq_group: SequenceGroup) -> None:
623 + if seq_group.request_id not in self.cross_block_tables:
624 + # Already freed or hasn't ben scheduled yet.
625 + return
626 + block_table = self.cross_block_tables[seq_group.request_id]
627 + self._free_block_table(block_table)
628 + del self.cross_block_tables[seq_group.request_id]
629 +
630 def reset(self) -> None:
631 + # Free decoder block tables
632 for block_table in self.block_tables.values():
633 self._free_block_table(block_table)
634 self.block_tables.clear()
635 + # Free cross-attention block tables
636 + for block_table in self.cross_block_tables.values():
637 + self._free_block_table(block_table)
638 + self.cross_block_tables.clear()
639
640 def get_block_table(self, seq: Sequence) -> List[int]:
641 block_table = self.block_tables[seq.seq_id]
642 return [block.block_number for block in block_table]
643
644 + def get_cross_block_table(self, seq_group: SequenceGroup) -> List[int]:
645 + block_table = self.cross_block_tables[seq_group.request_id]
646 + return [block.block_number for block in block_table]
647 +
648 def get_num_free_gpu_blocks(self) -> int:
649 return self.gpu_allocator.get_num_free_blocks()
650
vllm/core/block_manager_v2.py
+56 -9
@@ -5,11 +5,13 @@ from typing import Tuple
5
6 from vllm.core.block.block_table import BlockTable
7 from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
8 +from vllm.core.block.utils import check_no_caching_or_swa_for_blockmgr_encdec
9 from vllm.core.interfaces import AllocStatus, BlockSpaceManager
10 from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
11 from vllm.utils import Device
12
13 SeqId = int
14 +EncoderSeqId = str
15
16
17 class BlockSpaceManagerV2(BlockSpaceManager):
@@ -94,17 +96,26 @@ class BlockSpaceManagerV2(BlockSpaceManager):
96 )
97
98 self.block_tables: Dict[SeqId, BlockTable] = {}
99 + self.cross_block_tables: Dict[EncoderSeqId, BlockTable] = {}
100
101 def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
102 # FIXME(woosuk): Here we assume that all sequences in the group share
103 # the same prompt. This may not be true for preempted sequences.
101 - seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
104
105 + check_no_caching_or_swa_for_blockmgr_encdec(self, seq_group)
106 +
107 + seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
108 num_required_blocks = BlockTable.get_num_required_blocks(
109 seq.get_token_ids(),
110 block_size=self.block_size,
111 )
112
113 + if seq_group.is_encoder_decoder():
114 + num_required_blocks += BlockTable.get_num_required_blocks(
115 + seq_group.get_encoder_seq().get_token_ids(),
116 + block_size=self.block_size,
117 + )
118 +
119 if self.max_block_sliding_window is not None:
120 num_required_blocks = min(num_required_blocks,
121 self.max_block_sliding_window)
@@ -121,7 +132,19 @@ class BlockSpaceManagerV2(BlockSpaceManager):
132 else:
133 return AllocStatus.LATER
134
135 + def _allocate_sequence(self, seq: Sequence) -> BlockTable:
136 + block_table = BlockTable(
137 + block_size=self.block_size,
138 + block_allocator=self.block_allocator,
139 + max_block_sliding_window=self.max_block_sliding_window,
140 + )
141 + block_table.allocate(seq.get_token_ids())
142 +
143 + return block_table
144 +
145 def allocate(self, seq_group: SequenceGroup) -> None:
146 +
147 + # Allocate self-attention block tables for decoder sequences
148 waiting_seqs = seq_group.get_seqs(status=SequenceStatus.WAITING)
149 assert not (set(seq.seq_id for seq in waiting_seqs)
150 & self.block_tables.keys()), "block table already exists"
@@ -129,20 +152,29 @@ class BlockSpaceManagerV2(BlockSpaceManager):
152 # NOTE: Here we assume that all sequences in the group have the same
153 # prompt.
154 seq = waiting_seqs[0]
132 -
133 - block_table = BlockTable(
134 - block_size=self.block_size,
135 - block_allocator=self.block_allocator,
136 - max_block_sliding_window=self.max_block_sliding_window,
137 - )
138 -
139 - block_table.allocate(seq.get_token_ids())
155 + block_table: BlockTable = self._allocate_sequence(seq)
156 self.block_tables[seq.seq_id] = block_table
157
158 # Assign the block table for each sequence.
159 for seq in waiting_seqs[1:]:
160 self.block_tables[seq.seq_id] = block_table.fork()
161
162 + # Allocate cross-attention block table for encoder sequence
163 + #
164 + # NOTE: Here we assume that all sequences in the group have the same
165 + # encoder prompt.
166 + request_id = seq_group.request_id
167 +
168 + assert (request_id
169 + not in self.cross_block_tables), \
170 + "block table already exists"
171 +
172 + check_no_caching_or_swa_for_blockmgr_encdec(self, seq_group)
173 +
174 + if seq_group.is_encoder_decoder():
175 + block_table = self._allocate_sequence(seq_group.get_encoder_seq())
176 + self.cross_block_tables[request_id] = block_table
177 +
178 def can_append_slots(self, seq_group: SequenceGroup,
179 num_lookahead_slots: int) -> bool:
180 """Determine if there is enough space in the GPU KV cache to continue
@@ -197,12 +229,27 @@ class BlockSpaceManagerV2(BlockSpaceManager):
229 self.block_tables[seq.seq_id].free()
230 del self.block_tables[seq.seq_id]
231
232 + def free_cross(self, seq_group: SequenceGroup) -> None:
233 + request_id = seq_group.request_id
234 + if request_id not in self.cross_block_tables:
235 + # Already freed or hasn't been scheduled yet.
236 + return
237 + self.cross_block_tables[request_id].free()
238 + del self.cross_block_tables[request_id]
239 +
240 def get_block_table(self, seq: Sequence) -> List[int]:
241 assert seq.seq_id in self.block_tables
242 block_ids = self.block_tables[seq.seq_id].physical_block_ids
243 assert all(b is not None for b in block_ids)
244 return block_ids # type: ignore
245
246 + def get_cross_block_table(self, seq_group: SequenceGroup) -> List[int]:
247 + request_id = seq_group.request_id
248 + assert request_id in self.cross_block_tables
249 + block_ids = self.cross_block_tables[request_id].physical_block_ids
250 + assert all(b is not None for b in block_ids)
251 + return block_ids # type: ignore
252 +
253 def access_all_blocks_in_seq(self, seq: Sequence, now: float):
254 # Update the last accessed time of all the blocks accessed
255 # in this step.
vllm/sequence.py
+23
@@ -430,6 +430,8 @@ class SequenceGroup:
430 for an embedding model.
431 pooling_params: The pooling parameters used to generate the pooling
432 for an embedding model.
433 + encoder_seq: Optional, the single encoder sequence. Should be None
434 + unless you are working with an encoder/decoder model.
435 """
436
437 def __init__(
@@ -441,6 +443,7 @@ class SequenceGroup:
443 lora_request: Optional[LoRARequest] = None,
444 embeddings: Optional[List[float]] = None,
445 pooling_params: Optional[PoolingParams] = None,
446 + encoder_seq: Optional[Sequence] = None,
447 ) -> None:
448 self.request_id = request_id
449 self.seqs_dict = {seq.seq_id: seq for seq in seqs}
@@ -455,6 +458,7 @@ class SequenceGroup:
458 self.state = SequenceGroupState()
459 self.embeddings = embeddings
460 self.pooling_params = pooling_params
461 + self.encoder_seq = encoder_seq
462
463 @property
464 def prompt(self) -> Optional[str]:
@@ -538,6 +542,12 @@ class SequenceGroup:
542 seq for seq in self.seqs_dict.values() if seq.status == status
543 ]
544
545 + def is_encoder_decoder(self) -> bool:
546 + return self.encoder_seq is not None
547 +
548 + def get_encoder_seq(self) -> Optional[Sequence]:
549 + return self.encoder_seq
550 +
551 def get_unfinished_seqs(self) -> List[Sequence]:
552 return [
553 seq for seq in self.seqs_dict.values() if not seq.is_finished()
@@ -621,6 +631,15 @@ class SequenceGroupMetadata:
631 used in prefix caching.
632 state: Internal state tied to this sequence group.
633 multi_modal_data: Multi modal data.
634 + encoder_seq_data: Optional sequence data for encoder prompt
635 + (SequenceGroup.encoder_seq). Should be None
636 + unless you are working with an encoder/decoder
637 + model.
638 + cross_block_table: Optional cross-attention block table associated
639 + with the encoder prompt
640 + (SequenceGroup.encoder_seq). Should be None
641 + unless you are working with an encoder/decoder
642 + model.
643 """
644
645 def __init__(
@@ -637,6 +656,8 @@ class SequenceGroupMetadata:
656 computed_block_nums: Optional[List[int]] = None,
657 state: Optional[SequenceGroupState] = None,
658 multi_modal_data: Optional[MultiModalData] = None,
659 + encoder_seq_data: Optional[SequenceData] = None,
660 + cross_block_table: Optional[List[int]] = None,
661 ) -> None:
662 self.request_id = request_id
663 self.is_prompt = is_prompt
@@ -648,6 +669,8 @@ class SequenceGroupMetadata:
669 self.computed_block_nums = computed_block_nums
670 self.multi_modal_data = multi_modal_data
671 self.state = SequenceGroupState() if state is None else state
672 + self.encoder_seq_data = encoder_seq_data
673 + self.cross_block_table = cross_block_table
674 self._token_chunk_size = token_chunk_size
675 self.do_sample = do_sample
676