A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Core] Cross-attention KV caching and memory-management (towards eventual encoder/decoder model support) (#4837)
afeldman-nm committed
May 29, 2024 at 12:09 UTC
4238bc82f24d5887784b04a353ed93e2360623b4
7 files changed
+735
-69
tests/core/block/test_block_manager_v2.py
+153
-1
@@ -1,11 +1,13 @@
1
import pytest
2
3
+from vllm.core.block.utils import (STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE,
4
+ STR_NOT_IMPL_ENC_DEC_SWA)
5
from vllm.core.block_manager_v2 import BlockSpaceManagerV2
6
from vllm.core.interfaces import AllocStatus
7
from vllm.sequence import Logprob, SequenceStatus
8
from vllm.utils import chunk_list
9
8
-from ..utils import create_seq_group
10
+from ..utils import create_seq_group, create_seq_group_encoder_decoder
11
12
13
@pytest.mark.parametrize("block_size", [16])
@@ -52,6 +54,156 @@ def test_can_allocate_seq_group(block_size: int, num_seqs_per_group: int,
54
assert can_allocate_result == AllocStatus.LATER
55
56
57
+@pytest.mark.parametrize("block_size", [16])
58
+@pytest.mark.parametrize("num_gpu_blocks", [16, 80, 160])
59
+@pytest.mark.parametrize("num_seqs_per_group", [1, 4])
60
+@pytest.mark.parametrize("watermark", [0.0, 0.5])
61
+def test_can_allocate_seq_group_encoder_decoder(block_size: int,
62
+ num_seqs_per_group: int,
63
+ num_gpu_blocks: int,
64
+ watermark: float):
65
+ block_manager = BlockSpaceManagerV2(
66
+ block_size=block_size,
67
+ num_gpu_blocks=num_gpu_blocks,
68
+ num_cpu_blocks=1024,
69
+ watermark=watermark,
70
+ )
71
+ num_watermark_blocks = int(watermark * num_gpu_blocks)
72
+
73
+ num_output_blocks_per_seq = 1
74
+
75
+ # NOTE: This should be num_output_blocks_per_seq * num_seqs_per_group, but
76
+ # the current implementation assumes all seqs are new prompts / don't have
77
+ # different output lens.
78
+ num_output_blocks = num_output_blocks_per_seq
79
+
80
+ for bdx, num_prompt_blocks in enumerate(
81
+ range(1, num_gpu_blocks - num_output_blocks)):
82
+ num_cross_blocks_per_seq = num_prompt_blocks
83
+
84
+ seq_group = create_seq_group_encoder_decoder(
85
+ seq_prompt_len=block_size * num_prompt_blocks,
86
+ seq_output_lens=[
87
+ block_size * num_output_blocks_per_seq
88
+ for _ in range(num_seqs_per_group)
89
+ ],
90
+ request_id=str(bdx))
91
+
92
+ assert num_prompt_blocks + num_output_blocks <= num_gpu_blocks
93
+
94
+ can_allocate_result = block_manager.can_allocate(seq_group)
95
+
96
+ num_required_blocks = num_prompt_blocks + \
97
+ num_output_blocks + \
98
+ num_cross_blocks_per_seq
99
+
100
+ if num_gpu_blocks - num_required_blocks < num_watermark_blocks:
101
+ assert can_allocate_result == AllocStatus.NEVER
102
+ elif num_gpu_blocks >= num_required_blocks:
103
+ assert can_allocate_result == AllocStatus.OK
104
+ else:
105
+ assert can_allocate_result == AllocStatus.LATER
106
+
107
+
108
+@pytest.mark.parametrize("block_size", [16])
109
+@pytest.mark.parametrize("num_gpu_blocks", [16])
110
+@pytest.mark.parametrize("num_seqs_per_group", [1])
111
+@pytest.mark.parametrize("watermark", [0.0, 0.5])
112
+def test_can_allocate_encoder_decoder_fails_with_swa(block_size: int,
113
+ num_seqs_per_group: int,
114
+ num_gpu_blocks: int,
115
+ watermark: float):
116
+ '''
117
+ SWA short for Sliding Window Attention.
118
+
119
+ At time of writing block manager v2 does not support SWA.
120
+
121
+ However even when SWA is implemented for block manager v2,
122
+ there will still most likely be a separate workstream required
123
+ to enable SWA for encoder/decoder models.
124
+
125
+ Therefore this test enforces that one of the following cases
126
+ hold true:
127
+ 1. Block manager v2 does not support SWA at all (true at time of writing)
128
+ 2. Block manager v2 fails with NotImplementError when SWA is enabled
129
+ AND a SequenceGroup with an encoder sequence (i.e. in support of an
130
+ encoder/decoder model) is passed into can_allocate() as an argument
131
+
132
+ The setup for this test is stripped down version of
133
+ test_can_allocate_seq_group_encoder_decoder()
134
+ '''
135
+
136
+ with pytest.raises((NotImplementedError, AssertionError)) as exc_info:
137
+ block_manager = BlockSpaceManagerV2(
138
+ block_size=block_size,
139
+ num_gpu_blocks=num_gpu_blocks,
140
+ num_cpu_blocks=1024,
141
+ watermark=watermark,
142
+ sliding_window=5 # SWA
143
+ )
144
+
145
+ num_output_blocks_per_seq = 1
146
+ num_prompt_blocks = 1
147
+ num_output_blocks = num_output_blocks_per_seq
148
+ seq_group = create_seq_group_encoder_decoder(
149
+ seq_prompt_len=block_size * num_prompt_blocks,
150
+ seq_output_lens=[
151
+ block_size * num_output_blocks_per_seq
152
+ for _ in range(num_seqs_per_group)
153
+ ],
154
+ request_id="0")
155
+
156
+ assert num_prompt_blocks + num_output_blocks <= num_gpu_blocks
157
+ block_manager.can_allocate(seq_group)
158
+
159
+ # Assert that either
160
+ # 1. Block manager v2 constructor fails with assertion that sliding window
161
+ # is not yet supported (most likely near-term outcome at time of
162
+ # writing), or
163
+ # 2. can_allocate() fails with NotImplementedError due to combination of
164
+ # encoder/decoder and sliding window attention
165
+ if isinstance(exc_info.value, NotImplementedError):
166
+ assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_SWA
167
+ elif isinstance(exc_info.value, AssertionError):
168
+ assert str(exc_info.value) == "Sliding window not yet supported"
169
+
170
+
171
+@pytest.mark.parametrize("block_size", [16])
172
+@pytest.mark.parametrize("num_gpu_blocks", [16])
173
+@pytest.mark.parametrize("num_seqs_per_group", [1])
174
+@pytest.mark.parametrize("watermark", [0.0, 0.5])
175
+def test_can_allocate_encoder_decoder_fails_with_prefix_cache(
176
+ block_size: int, num_seqs_per_group: int, num_gpu_blocks: int,
177
+ watermark: float):
178
+
179
+ block_manager = BlockSpaceManagerV2(
180
+ block_size=block_size,
181
+ num_gpu_blocks=num_gpu_blocks,
182
+ num_cpu_blocks=1024,
183
+ watermark=watermark,
184
+ enable_caching=True # Prefix cache
185
+ )
186
+
187
+ num_output_blocks_per_seq = 1
188
+ num_prompt_blocks = 1
189
+ num_output_blocks = num_output_blocks_per_seq
190
+ seq_group = create_seq_group_encoder_decoder(
191
+ seq_prompt_len=block_size * num_prompt_blocks,
192
+ seq_output_lens=[
193
+ block_size * num_output_blocks_per_seq
194
+ for _ in range(num_seqs_per_group)
195
+ ],
196
+ request_id="0")
197
+
198
+ assert num_prompt_blocks + num_output_blocks <= num_gpu_blocks
199
+
200
+ # Assert that either can_allocate() fails with NotImplementedError
201
+ # due to combination of encoder/decoder and prefix cache
202
+ with pytest.raises(NotImplementedError) as exc_info:
203
+ block_manager.can_allocate(seq_group)
204
+ assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE
205
+
206
+
207
@pytest.mark.parametrize("block_size", [1, 8])
208
@pytest.mark.parametrize("prompt_len", [1, 7, 8])
209
@pytest.mark.parametrize("num_slots_to_append", [1, 8, 129])
tests/core/test_block_manager.py
+217
-3
@@ -6,13 +6,15 @@ import pytest
6
7
from vllm import SamplingParams
8
from vllm.block import PhysicalTokenBlock
9
+from vllm.core.block.utils import (STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE,
10
+ STR_NOT_IMPL_ENC_DEC_SWA)
11
from vllm.core.block_manager_v1 import (BlockSpaceManagerV1,
12
UncachedBlockAllocator)
13
from vllm.core.interfaces import AllocStatus
14
from vllm.sequence import Logprob, Sequence, SequenceGroup, SequenceStatus
15
from vllm.utils import Device
16
15
-from .utils import create_dummy_prompt
17
+from .utils import create_dummy_prompt, create_dummy_prompt_encoder_decoder
18
19
20
def test_block_allocator_allocate():
@@ -73,7 +75,7 @@ def test_allocate():
75
# Allocate same sequence group to all available gpu blocks.
76
for i in range(num_gpu_blocks):
77
_, seq_group = create_dummy_prompt(str(i), block_size)
76
- assert block_manager.can_allocate(seq_group)
78
+ assert block_manager.can_allocate(seq_group) == AllocStatus.OK
79
block_manager.allocate(seq_group)
80
assert block_manager.can_allocate(seq_group) != AllocStatus.OK
81
@@ -85,11 +87,107 @@ def test_allocate():
87
watermark=1 / num_gpu_blocks)
88
for i in range(num_gpu_blocks - 1):
89
_, seq_group = create_dummy_prompt(str(i), block_size)
88
- assert block_manager.can_allocate(seq_group)
90
+ assert block_manager.can_allocate(seq_group) == AllocStatus.OK
91
block_manager.allocate(seq_group)
92
assert block_manager.can_allocate(seq_group) != AllocStatus.OK
93
94
95
+def test_allocate_encoder_decoder():
96
+ block_size = 4
97
+ num_cpu_blocks = 4
98
+ num_gpu_blocks = 4
99
+ block_req_per_seq_group = 2
100
+ block_manager = BlockSpaceManagerV1(block_size,
101
+ num_cpu_blocks,
102
+ num_gpu_blocks,
103
+ watermark=0)
104
+
105
+ # Allocate same sequence group to all available gpu blocks.
106
+ for i in range(num_gpu_blocks // block_req_per_seq_group):
107
+ _, _, seq_group = create_dummy_prompt_encoder_decoder(
108
+ str(i),
109
+ decoder_prompt_length=block_size,
110
+ encoder_prompt_length=block_size)
111
+ assert block_manager.can_allocate(seq_group) == AllocStatus.OK
112
+ block_manager.allocate(seq_group)
113
+ assert block_manager.can_allocate(seq_group) != AllocStatus.OK
114
+
115
+ # Allocate same sequence group to all available gpu blocks.
116
+ # Use watermark to reserve one gpu block.
117
+ block_manager = BlockSpaceManagerV1(block_size,
118
+ num_cpu_blocks,
119
+ num_gpu_blocks,
120
+ watermark=1 / num_gpu_blocks)
121
+ for i in range((num_gpu_blocks - 1) // block_req_per_seq_group):
122
+ _, _, seq_group = create_dummy_prompt_encoder_decoder(
123
+ str(i),
124
+ decoder_prompt_length=block_size,
125
+ encoder_prompt_length=block_size)
126
+ assert block_manager.can_allocate(seq_group) == AllocStatus.OK
127
+ block_manager.allocate(seq_group)
128
+ assert block_manager.can_allocate(seq_group) != AllocStatus.OK
129
+
130
+
131
+def test_allocate_encoder_decoder_fails_with_swa():
132
+ # SWA short for sliding window attention
133
+
134
+ block_size = 4
135
+ num_cpu_blocks = 4
136
+ num_gpu_blocks = 4
137
+ block_manager = BlockSpaceManagerV1(block_size,
138
+ num_cpu_blocks,
139
+ num_gpu_blocks,
140
+ watermark=0,
141
+ sliding_window=5) # swa
142
+
143
+ # Allocate same sequence group to all available gpu blocks.
144
+ _, _, seq_group = create_dummy_prompt_encoder_decoder(
145
+ "0",
146
+ decoder_prompt_length=block_size,
147
+ encoder_prompt_length=block_size)
148
+
149
+ # Assert that can_allocate() fails due to SWA
150
+ with pytest.raises(NotImplementedError) as exc_info:
151
+ block_manager.can_allocate(seq_group)
152
+
153
+ assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_SWA
154
+
155
+ # Assert that allocate() fails due to SWA
156
+ with pytest.raises(NotImplementedError) as exc_info:
157
+ block_manager.allocate(seq_group)
158
+
159
+ assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_SWA
160
+
161
+
162
+def test_allocate_encoder_decoder_fails_with_prefix_caching():
163
+ block_size = 4
164
+ num_cpu_blocks = 4
165
+ num_gpu_blocks = 4
166
+ block_manager = BlockSpaceManagerV1(block_size,
167
+ num_cpu_blocks,
168
+ num_gpu_blocks,
169
+ watermark=0,
170
+ enable_caching=True) # Prefix cache
171
+
172
+ # Allocate same sequence group to all available gpu blocks.
173
+ _, _, seq_group = create_dummy_prompt_encoder_decoder(
174
+ "0",
175
+ decoder_prompt_length=block_size,
176
+ encoder_prompt_length=block_size)
177
+
178
+ # Assert that can_allocate() fails due to prefix caching
179
+ with pytest.raises(NotImplementedError) as exc_info:
180
+ block_manager.can_allocate(seq_group)
181
+
182
+ assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE
183
+
184
+ # Assert that allocate() fails due to prefix caching
185
+ with pytest.raises(NotImplementedError) as exc_info:
186
+ block_manager.allocate(seq_group)
187
+
188
+ assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE
189
+
190
+
191
def test_append_slot_single_seq():
192
block_size = 4
193
num_cpu_blocks = 4
@@ -244,6 +342,62 @@ def test_swap():
342
assert before_gpu_blocks == after_gpu_blocks + len(cpu_blocks)
343
344
345
+def test_swap_encoder_decoder():
346
+ block_size = 4
347
+ num_cpu_blocks = 4
348
+ num_gpu_blocks = 4
349
+ block_manager = BlockSpaceManagerV1(block_size,
350
+ num_cpu_blocks,
351
+ num_gpu_blocks,
352
+ watermark=0)
353
+
354
+ decoder_prompt, encoder_prompt, seq_group = \
355
+ create_dummy_prompt_encoder_decoder(
356
+ "1",
357
+ decoder_prompt_length=block_size,
358
+ encoder_prompt_length=block_size)
359
+ decoder_prompt.status = SequenceStatus.WAITING
360
+ encoder_prompt.status = SequenceStatus.WAITING
361
+ block_manager.allocate(seq_group)
362
+
363
+ # Emulate a forward pass by appending a single token.
364
+ # The block manager then knows how many unprocessed
365
+ # tokens will be written in the next forward pass.
366
+ token_id = 0
367
+ decoder_prompt.status = SequenceStatus.RUNNING
368
+ decoder_prompt.append_token_id(token_id, {token_id: Logprob(0.0)})
369
+
370
+ # Swap encoder/decoder seq group from GPU -> CPU.
371
+ decoder_gpu_blocks = block_manager.get_block_table(decoder_prompt)
372
+ cross_gpu_blocks = block_manager.get_cross_block_table(seq_group)
373
+ gpu_blocks = decoder_gpu_blocks + cross_gpu_blocks
374
+ assert block_manager.can_swap_out(seq_group)
375
+ before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
376
+ before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
377
+ mapping = block_manager.swap_out(seq_group)
378
+ assert [x[0] for x in mapping] == gpu_blocks
379
+ #assert list(mapping.keys()) == gpu_blocks
380
+ after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
381
+ after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
382
+ assert before_cpu_blocks == after_cpu_blocks + len(gpu_blocks)
383
+ assert before_gpu_blocks + len(gpu_blocks) == after_gpu_blocks
384
+ decoder_prompt.status = SequenceStatus.SWAPPED
385
+
386
+ # Swap encoder/decoder seq group from CPU -> GPU.
387
+ decoder_cpu_blocks = block_manager.get_block_table(decoder_prompt)
388
+ cross_cpu_blocks = block_manager.get_cross_block_table(seq_group)
389
+ cpu_blocks = decoder_cpu_blocks + cross_cpu_blocks
390
+ assert block_manager.can_swap_in(seq_group) == AllocStatus.OK
391
+ before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
392
+ before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
393
+ mapping = block_manager.swap_in(seq_group)
394
+ assert [x[0] for x in mapping] == cpu_blocks
395
+ after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
396
+ after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
397
+ assert before_cpu_blocks + len(cpu_blocks) == after_cpu_blocks
398
+ assert before_gpu_blocks == after_gpu_blocks + len(cpu_blocks)
399
+
400
+
401
def test_free():
402
block_size = 4
403
num_cpu_blocks = 4
@@ -268,6 +422,41 @@ def test_free():
422
block_manager.get_block_table(prompt)
423
424
425
+def test_free_encoder_decoder():
426
+ block_size = 4
427
+ num_cpu_blocks = 4
428
+ num_gpu_blocks = 4
429
+ block_manager = BlockSpaceManagerV1(block_size,
430
+ num_cpu_blocks,
431
+ num_gpu_blocks,
432
+ watermark=0)
433
+
434
+ decoder_prompt, encoder_prompt, seq_group = \
435
+ create_dummy_prompt_encoder_decoder(
436
+ "1",
437
+ decoder_prompt_length=block_size,
438
+ encoder_prompt_length=block_size)
439
+ block_manager.allocate(seq_group)
440
+
441
+ # Free allocated seq.
442
+ decoder_prompt_blocks = len(block_manager.get_block_table(decoder_prompt))
443
+ encoder_prompt_blocks = len(block_manager.get_cross_block_table(seq_group))
444
+ prompt_blocks = decoder_prompt_blocks + encoder_prompt_blocks
445
+ before_blocks = block_manager.get_num_free_gpu_blocks()
446
+ block_manager.free(decoder_prompt)
447
+ block_manager.free_cross(seq_group)
448
+ after_blocks = block_manager.get_num_free_gpu_blocks()
449
+ assert after_blocks == before_blocks + prompt_blocks
450
+
451
+ # Block table for freed encoder & decoder seq's are deleted.
452
+ with pytest.raises(KeyError):
453
+ block_manager.get_block_table(decoder_prompt)
454
+
455
+ # Block table for freed encoder & decoder seq's are deleted.
456
+ with pytest.raises(KeyError):
457
+ block_manager.get_block_table(encoder_prompt)
458
+
459
+
460
def test_reset():
461
block_size = 4
462
num_cpu_blocks = 4
@@ -289,6 +478,31 @@ def test_reset():
478
assert block_manager.get_num_free_gpu_blocks() == original_blocks
479
480
481
+def test_reset_encoder_decoder():
482
+ block_size = 4
483
+ num_cpu_blocks = 4
484
+ num_gpu_blocks = 4
485
+ block_req_per_seq_group = 2
486
+ block_manager = BlockSpaceManagerV1(block_size,
487
+ num_cpu_blocks,
488
+ num_gpu_blocks,
489
+ watermark=0)
490
+
491
+ # Allocate same seq group on all available gpu blocks.
492
+ original_blocks = block_manager.get_num_free_gpu_blocks()
493
+ for i in range(num_gpu_blocks // block_req_per_seq_group):
494
+ _, _, seq_group = create_dummy_prompt_encoder_decoder(
495
+ f"{i}",
496
+ decoder_prompt_length=block_size,
497
+ encoder_prompt_length=block_size)
498
+ block_manager.allocate(seq_group)
499
+ assert block_manager.get_num_free_gpu_blocks() == 0
500
+
501
+ # Resetting block manager frees all allocated blocks.
502
+ block_manager.reset()
503
+ assert block_manager.get_num_free_gpu_blocks() == original_blocks
504
+
505
+
506
def test_sliding_window_multi_seq():
507
"""
508
Tests that memory allocation and deallocation is handled
tests/core/utils.py
+98
-1
@@ -39,6 +39,52 @@ def create_dummy_prompt(
39
return prompt, seq_group
40
41
42
+def create_dummy_prompt_encoder_decoder(
43
+ request_id: str,
44
+ decoder_prompt_length: int,
45
+ encoder_prompt_length: int,
46
+ block_size: Optional[int] = None,
47
+ lora_request: Optional[LoRARequest] = None,
48
+ use_beam_search: bool = False,
49
+ best_of: int = 1,
50
+) -> Tuple[Sequence, SequenceGroup]:
51
+ if not block_size:
52
+ block_size = decoder_prompt_length
53
+
54
+ # Create dummy prompt sequence with tokens 0...block_size-1
55
+ # and prompt "0 ... block_size".
56
+ decoder_prompt_tokens = list(range(decoder_prompt_length))
57
+ decoder_prompt_str = " ".join([str(t) for t in decoder_prompt_tokens])
58
+
59
+ decoder_prompt = Sequence(int(request_id),
60
+ inputs={
61
+ "prompt": decoder_prompt_str,
62
+ "prompt_token_ids": decoder_prompt_tokens,
63
+ "multi_modal_data": None,
64
+ },
65
+ block_size=block_size)
66
+
67
+ encoder_prompt_tokens = list(reversed(list(range(encoder_prompt_length))))
68
+ encoder_prompt_str = " ".join([str(t) for t in encoder_prompt_tokens])
69
+ encoder_prompt = Sequence(int(request_id),
70
+ inputs={
71
+ "prompt": encoder_prompt_str,
72
+ "prompt_token_ids": encoder_prompt_tokens,
73
+ "multi_modal_data": None,
74
+ },
75
+ block_size=block_size)
76
+ seq_group = SequenceGroup(request_id=request_id,
77
+ seqs=[decoder_prompt],
78
+ sampling_params=SamplingParams(
79
+ use_beam_search=use_beam_search,
80
+ best_of=best_of),
81
+ arrival_time=time.time(),
82
+ lora_request=lora_request,
83
+ encoder_seq=encoder_prompt)
84
+
85
+ return decoder_prompt, encoder_prompt, seq_group
86
+
87
+
88
def create_seq_group(
89
seq_prompt_len: int = 1024,
90
seq_output_lens: Iterable[int] = (128, ),
@@ -82,5 +128,56 @@ def create_seq_group(
128
return seq_group
129
130
131
+def create_seq_group_encoder_decoder(
132
+ seq_prompt_len: int = 1024,
133
+ seq_output_lens: Iterable[int] = (128, ),
134
+ request_id: str = '0',
135
+ seq_id_start: int = 0,
136
+ sampling_params: Optional[SamplingParams] = None) -> SequenceGroup:
137
+
138
+ assert len(seq_output_lens) > 0
139
+
140
+ if sampling_params is None:
141
+ sampling_params = SamplingParams()
142
+
143
+ prompt_token_ids = [0] * seq_prompt_len
144
+
145
+ seqs = []
146
+ for seq_id_offset, output_len in enumerate(seq_output_lens):
147
+ seq = Sequence(
148
+ seq_id=seq_id_start + seq_id_offset,
149
+ inputs={
150
+ "prompt": "",
151
+ "prompt_token_ids": prompt_token_ids,
152
+ "multi_modal_data": None,
153
+ },
154
+ block_size=16,
155
+ )
156
+
157
+ for i in range(output_len):
158
+ seq.append_token_id(
159
+ token_id=i,
160
+ logprobs={i: Logprob(0.0)},
161
+ )
162
+ seqs.append(seq)
163
+
164
+ # Encoder sequence
165
+ encoder_seq = Sequence(
166
+ seq_id=seq_id_start + len(seq_output_lens),
167
+ inputs={
168
+ "prompt": "",
169
+ "prompt_token_ids": prompt_token_ids,
170
+ "multi_modal_data": None,
171
+ },
172
+ block_size=16,
173
+ )
174
+
175
+ return SequenceGroup(request_id=request_id,
176
+ seqs=seqs,
177
+ sampling_params=sampling_params,
178
+ arrival_time=time.time(),
179
+ encoder_seq=encoder_seq)
180
+
181
+
182
def round_up_to_next_block(seq_len: int, block_size: int) -> int:
86
- return (seq_len + block_size - 1) // block_size
183
+ return (seq_len + block_size - 1) // block_size
\ No newline at end of file
vllm/core/block/utils.py
new
+56
@@ -0,0 +1,56 @@
1
+"""Block manager utils."""
2
+from vllm.sequence import SequenceGroup
3
+
4
+# Exception strings for non-implemented block manager enc/dec scenarios
5
+
6
+STR_NOT_IMPL_ENC_DEC_SWA = \
7
+ "Sliding window attention for encoder/decoder models " + \
8
+ "is not currently supported."
9
+
10
+STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE = \
11
+ "Prefix caching for encoder/decoder models " + \
12
+ "is not currently supported."
13
+
14
+
15
+def _get_block_mgr_sliding_window_attr(block_mgr):
16
+ '''
17
+ BlockManagerV1 and BlockManagerV2 have slightly different
18
+ members related to sliding window attention (SWA). This
19
+ function extracts the appropriate member to use for determining
20
+ whether SWA is enabled.
21
+
22
+ Arguments:
23
+
24
+ * block_mgr: BlockManagerV1 or BlockManagerV2 instance
25
+ '''
26
+
27
+ if hasattr(block_mgr, 'block_sliding_window'):
28
+ return block_mgr.block_sliding_window
29
+ if hasattr(block_mgr, 'max_block_sliding_window'):
30
+ return block_mgr.max_block_sliding_window
31
+
32
+ raise AttributeError("Block manager instance has neither " + \
33
+ "block_sliding_window nor " + \
34
+ "max_block_sliding_window attributes.")
35
+
36
+
37
+def check_no_caching_or_swa_for_blockmgr_encdec(
38
+ block_mgr, seq_group: SequenceGroup) -> None:
39
+ '''
40
+ Enforce that prefix caching & sliding-window attention (SWA)
41
+ are currently unsupported *specifically* for encoder/decoder models.
42
+
43
+ Raises NotImplementedError if unsupported scenario is detected.
44
+
45
+ Arguments:
46
+
47
+ * block_mgr: BlockSpaceManager instance
48
+ * seq_group: SequenceGroup passed to block_mgr
49
+ '''
50
+
51
+ if seq_group.is_encoder_decoder():
52
+ if _get_block_mgr_sliding_window_attr(block_mgr) is not None:
53
+ raise NotImplementedError(STR_NOT_IMPL_ENC_DEC_SWA)
54
+
55
+ if block_mgr.enable_caching:
56
+ raise NotImplementedError(STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE)
vllm/core/block_manager_v1.py
+132
-55
@@ -8,6 +8,7 @@ from typing import Sequence as GenericSequence
8
from typing import Set, Tuple
9
10
from vllm.block import BlockTable, PhysicalTokenBlock
11
+from vllm.core.block.utils import check_no_caching_or_swa_for_blockmgr_encdec
12
from vllm.core.evictor_v1 import EvictionPolicy, Evictor, make_evictor
13
from vllm.core.interfaces import AllocStatus, BlockSpaceManager
14
from vllm.logger import init_logger
@@ -255,14 +256,30 @@ class BlockSpaceManagerV1(BlockSpaceManager):
256
Device.CPU, block_size, num_cpu_blocks)
257
# Mapping: seq_id -> BlockTable.
258
self.block_tables: Dict[int, BlockTable] = {}
259
+ # Mapping: req_id -> BlockTable
260
+ # Note that each SequenceGroup has a unique
261
+ # request ID
262
+ self.cross_block_tables: Dict[str, BlockTable] = {}
263
+
264
+ def _get_seq_num_required_blocks(self, seq: Sequence) -> int:
265
+ return 0 if seq is None \
266
+ else len(seq.logical_token_blocks)
267
268
def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
269
# FIXME(woosuk): Here we assume that all sequences in the group share
270
# the same prompt. This may not be true for preempted sequences.
262
- seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
263
- num_required_blocks = len(seq.logical_token_blocks)
271
+
272
+ check_no_caching_or_swa_for_blockmgr_encdec(self, seq_group)
273
+
274
+ self_num_required_blocks = self._get_seq_num_required_blocks(
275
+ seq_group.get_seqs(status=SequenceStatus.WAITING)[0])
276
+ cross_num_required_blocks = self._get_seq_num_required_blocks(
277
+ seq_group.get_encoder_seq())
278
+ num_required_blocks = self_num_required_blocks + \
279
+ cross_num_required_blocks
280
281
if self.block_sliding_window is not None:
282
+
283
num_required_blocks = min(num_required_blocks,
284
self.block_sliding_window)
285
num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
@@ -276,11 +293,10 @@ class BlockSpaceManagerV1(BlockSpaceManager):
293
else:
294
return AllocStatus.LATER
295
279
- def allocate(self, seq_group: SequenceGroup) -> None:
280
- # NOTE: Here we assume that all sequences in the group have the same
281
- # prompt.
282
- seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
283
-
296
+ def _allocate_sequence(self, \
297
+ seq: Sequence, \
298
+ ref_count: int, \
299
+ is_encoder_decoder: bool = True) -> BlockTable:
300
# Allocate new physical token blocks that will store the prompt tokens.
301
num_prompt_blocks = len(seq.logical_token_blocks)
302
@@ -290,21 +306,46 @@ class BlockSpaceManagerV1(BlockSpaceManager):
306
and logical_idx >= self.block_sliding_window):
307
block = block_table[logical_idx % self.block_sliding_window]
308
# Set the reference counts of the token blocks.
293
- block.ref_count = seq_group.num_seqs()
294
- elif self.enable_caching:
309
+ block.ref_count = ref_count
310
+ elif not is_encoder_decoder and self.enable_caching:
311
block = self.gpu_allocator.allocate(
312
seq.hash_of_block(logical_idx),
313
seq.num_hashed_tokens_of_block(logical_idx))
314
else:
315
block = self.gpu_allocator.allocate()
316
# Set the reference counts of the token blocks.
301
- block.ref_count = seq_group.num_seqs()
317
+ block.ref_count = ref_count
318
block_table.append(block)
319
304
- # Assign the block table for each sequence.
320
+ return block_table
321
+
322
+ def allocate(self, seq_group: SequenceGroup) -> None:
323
+ is_encoder_decoder = seq_group.is_encoder_decoder()
324
+ check_no_caching_or_swa_for_blockmgr_encdec(self, seq_group)
325
+
326
+ # Allocate decoder sequences
327
+ #
328
+ # NOTE: Here we assume that all sequences in the group have the same
329
+ # decoder prompt.
330
+ seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
331
+ block_table: BlockTable = \
332
+ self._allocate_sequence(seq,
333
+ seq_group.num_seqs(),
334
+ is_encoder_decoder)
335
+
336
+ # Assign the self-attention block tables for each sequence.
337
for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
338
self.block_tables[seq.seq_id] = block_table.copy()
339
340
+ # Allocate encoder sequence
341
+ if is_encoder_decoder:
342
+ # A SequenceGroup has only a single encoder sequence (at most),
343
+ # thus allocate with a ref count of 1
344
+ block_table = self._allocate_sequence(seq_group.get_encoder_seq(),
345
+ 1, is_encoder_decoder)
346
+ # Assign the cross-attention block table for the SequenceGroup.
347
+ self.cross_block_tables[seq_group.request_id] = block_table
348
+
349
def can_append_slots(self,
350
seq_group: SequenceGroup,
351
num_lookahead_slots: int = 0) -> bool:
@@ -443,13 +484,18 @@ class BlockSpaceManagerV1(BlockSpaceManager):
484
485
def _get_physical_blocks(
486
self, seq_group: SequenceGroup) -> List[PhysicalTokenBlock]:
487
+
488
# NOTE: Here, we assume that the physical blocks are only shared by
489
# the sequences in the same group.
490
+ request_id = seq_group.request_id
491
blocks: Set[PhysicalTokenBlock] = set()
492
for seq in seq_group.get_seqs():
493
if seq.is_finished():
494
continue
495
blocks.update(self.block_tables[seq.seq_id])
496
+ # Cross-attention blocks
497
+ if seq_group.is_encoder_decoder():
498
+ blocks.update(self.cross_block_tables[request_id])
499
return list(blocks)
500
501
def can_swap_in(self,
@@ -457,8 +503,11 @@ class BlockSpaceManagerV1(BlockSpaceManager):
503
num_lookahead_slots: int = 0) -> AllocStatus:
504
assert (num_lookahead_slots == 0
505
), "BlockSpaceManagerV1 does not support lookahead allocation"
506
+
507
blocks = self._get_physical_blocks(seq_group)
508
num_swapped_seqs = seq_group.num_seqs(status=SequenceStatus.SWAPPED)
509
+ if seq_group.is_encoder_decoder():
510
+ num_swapped_seqs += 1
511
num_free_blocks = self.gpu_allocator.get_num_free_blocks()
512
# NOTE: Conservatively, we assume that every sequence will allocate
513
# at least one free block right after the swap-in.
@@ -471,70 +520,81 @@ class BlockSpaceManagerV1(BlockSpaceManager):
520
else:
521
return AllocStatus.LATER
522
523
+ def _swap_block_table(
524
+ self, block_table: BlockTable, src_allocator: BlockAllocatorBase,
525
+ dest_allocator: BlockAllocatorBase,
526
+ mapping: Dict[PhysicalTokenBlock,
527
+ PhysicalTokenBlock]) -> BlockTable:
528
+ new_block_table = []
529
+
530
+ for from_block in block_table:
531
+ if from_block in mapping:
532
+ to_block = mapping[from_block]
533
+ to_block.ref_count += 1
534
+ else:
535
+ to_block = dest_allocator.allocate(
536
+ from_block.block_hash, from_block.num_hashed_tokens)
537
+ mapping[from_block] = to_block
538
+ new_block_table.append(to_block)
539
+ # Free the source block swapped in to destination.
540
+ src_allocator.free(from_block)
541
+
542
+ return new_block_table
543
+
544
def swap_in(self,
545
seq_group: SequenceGroup,
546
num_lookahead_slots: int = 0) -> List[Tuple[int, int]]:
547
assert (num_lookahead_slots == 0
548
), "BlockSpaceManagerV1 does not support lookahead allocation"
549
550
+ request_id = seq_group.request_id
551
+
552
# CPU block -> GPU block.
553
# dict is efficient in lookup `if cpu_block in mapping`
554
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
555
for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
484
- new_block_table: BlockTable = []
485
- block_table = self.block_tables[seq.seq_id]
486
-
487
- for cpu_block in block_table:
488
- if cpu_block in mapping:
489
- gpu_block = mapping[cpu_block]
490
- gpu_block.ref_count += 1
491
- else:
492
- gpu_block = self.gpu_allocator.allocate(
493
- cpu_block.block_hash, cpu_block.num_hashed_tokens)
494
- mapping[cpu_block] = gpu_block
495
- new_block_table.append(gpu_block)
496
- # Free the CPU block swapped in to GPU.
497
- self.cpu_allocator.free(cpu_block)
498
- self.block_tables[seq.seq_id] = new_block_table
499
-
500
- block_number_mapping = {
501
- cpu_block.block_number: gpu_block.block_number
502
- for cpu_block, gpu_block in mapping.items()
503
- }
504
- # convert to list of tuples once here
505
- return list(block_number_mapping.items())
556
+ self.block_tables[seq.seq_id] = \
557
+ self._swap_block_table(self.block_tables[seq.seq_id],
558
+ self.cpu_allocator,
559
+ self.gpu_allocator,
560
+ mapping)
561
+
562
+ if seq_group.is_encoder_decoder():
563
+ self.cross_block_tables[request_id] = \
564
+ self._swap_block_table(self.cross_block_tables[request_id],
565
+ self.cpu_allocator,
566
+ self.gpu_allocator,
567
+ mapping)
568
+
569
+ return [(cpu_block.block_number, gpu_block.block_number)
570
+ for cpu_block, gpu_block in mapping.items()]
571
572
def can_swap_out(self, seq_group: SequenceGroup) -> bool:
573
blocks = self._get_physical_blocks(seq_group)
574
return len(blocks) <= self.cpu_allocator.get_num_free_blocks()
575
576
def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
577
+ request_id = seq_group.request_id
578
+
579
# GPU block -> CPU block.
580
# dict is efficient in lookup `if gpu_block in mapping`
581
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
582
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
516
- new_block_table: BlockTable = []
517
- block_table = self.block_tables[seq.seq_id]
518
-
519
- for gpu_block in block_table:
520
- if gpu_block in mapping:
521
- cpu_block = mapping[gpu_block]
522
- cpu_block.ref_count += 1
523
- else:
524
- cpu_block = self.cpu_allocator.allocate(
525
- gpu_block.block_hash, gpu_block.num_hashed_tokens)
526
- mapping[gpu_block] = cpu_block
527
- new_block_table.append(cpu_block)
528
- # Free the GPU block swapped out to CPU.
529
- self.gpu_allocator.free(gpu_block)
530
- self.block_tables[seq.seq_id] = new_block_table
531
-
532
- block_number_mapping = {
533
- gpu_block.block_number: cpu_block.block_number
534
- for gpu_block, cpu_block in mapping.items()
535
- }
536
- # convert to list of tuples once here
537
- return list(block_number_mapping.items())
583
+ self.block_tables[seq.seq_id] = \
584
+ self._swap_block_table(self.block_tables[seq.seq_id],
585
+ self.gpu_allocator,
586
+ self.cpu_allocator,
587
+ mapping)
588
+
589
+ if seq_group.is_encoder_decoder():
590
+ self.cross_block_tables[request_id] = \
591
+ self._swap_block_table(self.cross_block_tables[request_id],
592
+ self.gpu_allocator,
593
+ self.cpu_allocator,
594
+ mapping)
595
+
596
+ return [(cpu_block.block_number, gpu_block.block_number)
597
+ for cpu_block, gpu_block in mapping.items()]
598
599
def _free_block_table(self, block_table: BlockTable) -> None:
600
# when using a sliding window, each seq will only use up
@@ -559,15 +619,32 @@ class BlockSpaceManagerV1(BlockSpaceManager):
619
self._free_block_table(block_table)
620
del self.block_tables[seq.seq_id]
621
622
+ def free_cross(self, seq_group: SequenceGroup) -> None:
623
+ if seq_group.request_id not in self.cross_block_tables:
624
+ # Already freed or hasn't ben scheduled yet.
625
+ return
626
+ block_table = self.cross_block_tables[seq_group.request_id]
627
+ self._free_block_table(block_table)
628
+ del self.cross_block_tables[seq_group.request_id]
629
+
630
def reset(self) -> None:
631
+ # Free decoder block tables
632
for block_table in self.block_tables.values():
633
self._free_block_table(block_table)
634
self.block_tables.clear()
635
+ # Free cross-attention block tables
636
+ for block_table in self.cross_block_tables.values():
637
+ self._free_block_table(block_table)
638
+ self.cross_block_tables.clear()
639
640
def get_block_table(self, seq: Sequence) -> List[int]:
641
block_table = self.block_tables[seq.seq_id]
642
return [block.block_number for block in block_table]
643
644
+ def get_cross_block_table(self, seq_group: SequenceGroup) -> List[int]:
645
+ block_table = self.cross_block_tables[seq_group.request_id]
646
+ return [block.block_number for block in block_table]
647
+
648
def get_num_free_gpu_blocks(self) -> int:
649
return self.gpu_allocator.get_num_free_blocks()
650
vllm/core/block_manager_v2.py
+56
-9
@@ -5,11 +5,13 @@ from typing import Tuple
5
6
from vllm.core.block.block_table import BlockTable
7
from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
8
+from vllm.core.block.utils import check_no_caching_or_swa_for_blockmgr_encdec
9
from vllm.core.interfaces import AllocStatus, BlockSpaceManager
10
from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
11
from vllm.utils import Device
12
13
SeqId = int
14
+EncoderSeqId = str
15
16
17
class BlockSpaceManagerV2(BlockSpaceManager):
@@ -94,17 +96,26 @@ class BlockSpaceManagerV2(BlockSpaceManager):
96
)
97
98
self.block_tables: Dict[SeqId, BlockTable] = {}
99
+ self.cross_block_tables: Dict[EncoderSeqId, BlockTable] = {}
100
101
def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
102
# FIXME(woosuk): Here we assume that all sequences in the group share
103
# the same prompt. This may not be true for preempted sequences.
101
- seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
104
105
+ check_no_caching_or_swa_for_blockmgr_encdec(self, seq_group)
106
+
107
+ seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
108
num_required_blocks = BlockTable.get_num_required_blocks(
109
seq.get_token_ids(),
110
block_size=self.block_size,
111
)
112
113
+ if seq_group.is_encoder_decoder():
114
+ num_required_blocks += BlockTable.get_num_required_blocks(
115
+ seq_group.get_encoder_seq().get_token_ids(),
116
+ block_size=self.block_size,
117
+ )
118
+
119
if self.max_block_sliding_window is not None:
120
num_required_blocks = min(num_required_blocks,
121
self.max_block_sliding_window)
@@ -121,7 +132,19 @@ class BlockSpaceManagerV2(BlockSpaceManager):
132
else:
133
return AllocStatus.LATER
134
135
+ def _allocate_sequence(self, seq: Sequence) -> BlockTable:
136
+ block_table = BlockTable(
137
+ block_size=self.block_size,
138
+ block_allocator=self.block_allocator,
139
+ max_block_sliding_window=self.max_block_sliding_window,
140
+ )
141
+ block_table.allocate(seq.get_token_ids())
142
+
143
+ return block_table
144
+
145
def allocate(self, seq_group: SequenceGroup) -> None:
146
+
147
+ # Allocate self-attention block tables for decoder sequences
148
waiting_seqs = seq_group.get_seqs(status=SequenceStatus.WAITING)
149
assert not (set(seq.seq_id for seq in waiting_seqs)
150
& self.block_tables.keys()), "block table already exists"
@@ -129,20 +152,29 @@ class BlockSpaceManagerV2(BlockSpaceManager):
152
# NOTE: Here we assume that all sequences in the group have the same
153
# prompt.
154
seq = waiting_seqs[0]
132
-
133
- block_table = BlockTable(
134
- block_size=self.block_size,
135
- block_allocator=self.block_allocator,
136
- max_block_sliding_window=self.max_block_sliding_window,
137
- )
138
-
139
- block_table.allocate(seq.get_token_ids())
155
+ block_table: BlockTable = self._allocate_sequence(seq)
156
self.block_tables[seq.seq_id] = block_table
157
158
# Assign the block table for each sequence.
159
for seq in waiting_seqs[1:]:
160
self.block_tables[seq.seq_id] = block_table.fork()
161
162
+ # Allocate cross-attention block table for encoder sequence
163
+ #
164
+ # NOTE: Here we assume that all sequences in the group have the same
165
+ # encoder prompt.
166
+ request_id = seq_group.request_id
167
+
168
+ assert (request_id
169
+ not in self.cross_block_tables), \
170
+ "block table already exists"
171
+
172
+ check_no_caching_or_swa_for_blockmgr_encdec(self, seq_group)
173
+
174
+ if seq_group.is_encoder_decoder():
175
+ block_table = self._allocate_sequence(seq_group.get_encoder_seq())
176
+ self.cross_block_tables[request_id] = block_table
177
+
178
def can_append_slots(self, seq_group: SequenceGroup,
179
num_lookahead_slots: int) -> bool:
180
"""Determine if there is enough space in the GPU KV cache to continue
@@ -197,12 +229,27 @@ class BlockSpaceManagerV2(BlockSpaceManager):
229
self.block_tables[seq.seq_id].free()
230
del self.block_tables[seq.seq_id]
231
232
+ def free_cross(self, seq_group: SequenceGroup) -> None:
233
+ request_id = seq_group.request_id
234
+ if request_id not in self.cross_block_tables:
235
+ # Already freed or hasn't been scheduled yet.
236
+ return
237
+ self.cross_block_tables[request_id].free()
238
+ del self.cross_block_tables[request_id]
239
+
240
def get_block_table(self, seq: Sequence) -> List[int]:
241
assert seq.seq_id in self.block_tables
242
block_ids = self.block_tables[seq.seq_id].physical_block_ids
243
assert all(b is not None for b in block_ids)
244
return block_ids # type: ignore
245
246
+ def get_cross_block_table(self, seq_group: SequenceGroup) -> List[int]:
247
+ request_id = seq_group.request_id
248
+ assert request_id in self.cross_block_tables
249
+ block_ids = self.cross_block_tables[request_id].physical_block_ids
250
+ assert all(b is not None for b in block_ids)
251
+ return block_ids # type: ignore
252
+
253
def access_all_blocks_in_seq(self, seq: Sequence, now: float):
254
# Update the last accessed time of all the blocks accessed
255
# in this step.
vllm/sequence.py
+23
@@ -430,6 +430,8 @@ class SequenceGroup:
430
for an embedding model.
431
pooling_params: The pooling parameters used to generate the pooling
432
for an embedding model.
433
+ encoder_seq: Optional, the single encoder sequence. Should be None
434
+ unless you are working with an encoder/decoder model.
435
"""
436
437
def __init__(
@@ -441,6 +443,7 @@ class SequenceGroup:
443
lora_request: Optional[LoRARequest] = None,
444
embeddings: Optional[List[float]] = None,
445
pooling_params: Optional[PoolingParams] = None,
446
+ encoder_seq: Optional[Sequence] = None,
447
) -> None:
448
self.request_id = request_id
449
self.seqs_dict = {seq.seq_id: seq for seq in seqs}
@@ -455,6 +458,7 @@ class SequenceGroup:
458
self.state = SequenceGroupState()
459
self.embeddings = embeddings
460
self.pooling_params = pooling_params
461
+ self.encoder_seq = encoder_seq
462
463
@property
464
def prompt(self) -> Optional[str]:
@@ -538,6 +542,12 @@ class SequenceGroup:
542
seq for seq in self.seqs_dict.values() if seq.status == status
543
]
544
545
+ def is_encoder_decoder(self) -> bool:
546
+ return self.encoder_seq is not None
547
+
548
+ def get_encoder_seq(self) -> Optional[Sequence]:
549
+ return self.encoder_seq
550
+
551
def get_unfinished_seqs(self) -> List[Sequence]:
552
return [
553
seq for seq in self.seqs_dict.values() if not seq.is_finished()
@@ -621,6 +631,15 @@ class SequenceGroupMetadata:
631
used in prefix caching.
632
state: Internal state tied to this sequence group.
633
multi_modal_data: Multi modal data.
634
+ encoder_seq_data: Optional sequence data for encoder prompt
635
+ (SequenceGroup.encoder_seq). Should be None
636
+ unless you are working with an encoder/decoder
637
+ model.
638
+ cross_block_table: Optional cross-attention block table associated
639
+ with the encoder prompt
640
+ (SequenceGroup.encoder_seq). Should be None
641
+ unless you are working with an encoder/decoder
642
+ model.
643
"""
644
645
def __init__(
@@ -637,6 +656,8 @@ class SequenceGroupMetadata:
656
computed_block_nums: Optional[List[int]] = None,
657
state: Optional[SequenceGroupState] = None,
658
multi_modal_data: Optional[MultiModalData] = None,
659
+ encoder_seq_data: Optional[SequenceData] = None,
660
+ cross_block_table: Optional[List[int]] = None,
661
) -> None:
662
self.request_id = request_id
663
self.is_prompt = is_prompt
@@ -648,6 +669,8 @@ class SequenceGroupMetadata:
669
self.computed_block_nums = computed_block_nums
670
self.multi_modal_data = multi_modal_data
671
self.state = SequenceGroupState() if state is None else state
672
+ self.encoder_seq_data = encoder_seq_data
673
+ self.cross_block_table = cross_block_table
674
self._token_chunk_size = token_chunk_size
675
self.do_sample = do_sample
676