A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Performance] Optimize e2e overheads: Reduce python allocations (#7162)
Alexander Matveev committed
Aug 9, 2024 at 00:34 UTC
e02ac5561748306186aaeaad6dad4c89484a2b45
11 files changed
+550
-125
vllm/attention/backends/flash_attn.py
+5
-1
@@ -259,7 +259,11 @@ class FlashAttentionMetadataBuilder(
259
block_table = block_tables[seq_id]
260
elif ((chunked_prefill_enabled or not is_prompt)
261
and block_tables is not None):
262
- block_table = block_tables[seq_id][-curr_sliding_window_block:]
262
+ if curr_sliding_window_block == 0:
263
+ block_table = block_tables[seq_id]
264
+ else:
265
+ block_table = block_tables[seq_id][
266
+ -curr_sliding_window_block:]
267
self.block_tables.append(block_table)
268
269
# Compute slot mapping.
vllm/attention/backends/utils.py
+10
-2
@@ -68,13 +68,21 @@ def compute_slot_mapping(is_profile_run: bool, slot_mapping: List[int],
68
# tokens are masked and the slot mapping will be
69
# [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
70
block_table = block_tables[seq_id]
71
- slot_mapping.extend([PAD_SLOT_ID] * max(0, start_idx - context_len))
72
- for i in range(max(start_idx, context_len), seq_len):
71
+
72
+ def add_slot(i):
73
block_number = block_table[i // block_size]
74
block_offset = i % block_size
75
slot = block_number * block_size + block_offset
76
slot_mapping.append(slot)
77
78
+ if start_idx == 0 and (seq_len - context_len) == 1:
79
+ # Optimization for common-case of decoding next token
80
+ add_slot(seq_len - 1)
81
+ else:
82
+ slot_mapping.extend([PAD_SLOT_ID] * max(0, start_idx - context_len))
83
+ for i in range(max(start_idx, context_len), seq_len):
84
+ add_slot(i)
85
+
86
87
TAttentionMetadata = TypeVar("TAttentionMetadata", bound='AttentionMetadata')
88
vllm/block.py
+45
-3
@@ -1,5 +1,5 @@
1
"""Token blocks."""
2
-from typing import List
2
+from typing import List, Optional
3
4
from vllm.utils import Device
5
@@ -37,5 +37,47 @@ class PhysicalTokenBlock:
37
f'computed={self.computed})')
38
39
40
-# Mapping: logical block number -> physical block.
41
-BlockTable = List[PhysicalTokenBlock]
40
+class BlockTable:
41
+ """Holds a list of blocks with caching of their associated block_ids
42
+ """
43
+
44
+ def __init__(self, blocks: Optional[List[PhysicalTokenBlock]] = None):
45
+ self._blocks: List[PhysicalTokenBlock] = []
46
+ self._block_ids: List[int] = []
47
+
48
+ if blocks is not None:
49
+ for block in blocks:
50
+ self.append(block)
51
+
52
+ def append(self, block: PhysicalTokenBlock):
53
+ self._blocks.append(block)
54
+ self._block_ids.append(block.block_number)
55
+
56
+ def __len__(self) -> int:
57
+ return len(self._blocks)
58
+
59
+ def __getitem__(self, key):
60
+ return self._blocks[key]
61
+
62
+ def __setitem__(self, key, value):
63
+ if isinstance(key, slice):
64
+ blocks = value
65
+ self._blocks[key] = blocks
66
+ self._block_ids[key] = [b.block_number for b in blocks]
67
+ else:
68
+ block = value
69
+ self._blocks[key] = block
70
+ self._block_ids[key] = block.block_number
71
+
72
+ def reset(self):
73
+ self._blocks = []
74
+ self._block_ids = []
75
+
76
+ def copy(self) -> "BlockTable":
77
+ return BlockTable(self._blocks)
78
+
79
+ def list(self) -> List[PhysicalTokenBlock]:
80
+ return self._blocks
81
+
82
+ def ids(self) -> List[int]:
83
+ return self._block_ids
vllm/core/block_manager_v1.py
+15
-12
@@ -170,7 +170,7 @@ class UncachedBlockAllocator(BlockAllocatorBase):
170
self.num_blocks = num_blocks
171
172
# Initialize the free blocks.
173
- self.free_blocks: BlockTable = []
173
+ self.free_blocks: List[PhysicalTokenBlock] = []
174
for i in range(num_blocks):
175
block = PhysicalTokenBlock(device=device,
176
block_number=i,
@@ -256,6 +256,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
256
Device.CPU, block_size, num_cpu_blocks)
257
# Mapping: seq_id -> BlockTable.
258
self.block_tables: Dict[int, BlockTable] = {}
259
+
260
# Mapping: req_id -> BlockTable
261
# Note that each SequenceGroup has a unique
262
# request ID
@@ -299,7 +300,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
300
# Allocate new physical token blocks that will store the prompt tokens.
301
num_prompt_blocks = seq.n_blocks
302
302
- block_table: BlockTable = []
303
+ block_table: BlockTable = BlockTable()
304
for logical_idx in range(num_prompt_blocks):
305
if (self.block_sliding_window is not None
306
and logical_idx >= self.block_sliding_window):
@@ -326,15 +327,19 @@ class BlockSpaceManagerV1(BlockSpaceManager):
327
#
328
# NOTE: Here we assume that all sequences in the group have the same
329
# decoder prompt.
329
- seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
330
+ wait_seqs = seq_group.get_seqs(status=SequenceStatus.WAITING)
331
+ seq = wait_seqs[0]
332
block_table: BlockTable = \
333
self._allocate_sequence(seq,
334
seq_group.num_seqs(),
335
is_encoder_decoder)
336
337
# Assign the self-attention block tables for each sequence.
336
- for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
337
- self.block_tables[seq.seq_id] = block_table.copy()
338
+ if len(wait_seqs) == 1:
339
+ self.block_tables[wait_seqs[0].seq_id] = block_table
340
+ else:
341
+ for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
342
+ self.block_tables[seq.seq_id] = block_table.copy()
343
344
# Allocate encoder sequence
345
if is_encoder_decoder:
@@ -476,6 +481,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
481
return
482
src_block_table = self.block_tables[parent_seq.seq_id]
483
self.block_tables[child_seq.seq_id] = src_block_table.copy()
484
+
485
# When using a sliding window, blocks will be eventually reused.
486
# In this case the block tables will contain repeated blocks.
487
# When forking, we must make sure that each block's `ref_count`
@@ -527,7 +533,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
533
dest_allocator: BlockAllocatorBase,
534
mapping: Dict[PhysicalTokenBlock,
535
PhysicalTokenBlock]) -> BlockTable:
530
- new_block_table = []
536
+ new_block_table: BlockTable = BlockTable()
537
538
for from_block in block_table:
539
if from_block in mapping:
@@ -553,8 +559,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
559
for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
560
self.block_tables[seq.seq_id] = \
561
self._swap_block_table(self.block_tables[seq.seq_id],
556
- self.cpu_allocator,
557
- self.gpu_allocator,
562
+ self.cpu_allocator, self.gpu_allocator,
563
mapping)
564
565
if seq_group.is_encoder_decoder():
@@ -580,8 +585,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
585
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
586
self.block_tables[seq.seq_id] = \
587
self._swap_block_table(self.block_tables[seq.seq_id],
583
- self.gpu_allocator,
584
- self.cpu_allocator,
588
+ self.gpu_allocator, self.cpu_allocator,
589
mapping)
590
591
if seq_group.is_encoder_decoder():
@@ -636,8 +640,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
640
self.cross_block_tables.clear()
641
642
def get_block_table(self, seq: Sequence) -> List[int]:
639
- block_table = self.block_tables[seq.seq_id]
640
- return [block.block_number for block in block_table]
643
+ return self.block_tables[seq.seq_id].ids()
644
645
def get_cross_block_table(self, seq_group: SequenceGroup) -> List[int]:
646
block_table = self.cross_block_tables[seq_group.request_id]
vllm/core/scheduler.py
+127
-44
@@ -13,6 +13,7 @@ from vllm.lora.request import LoRARequest
13
from vllm.prompt_adapter.request import PromptAdapterRequest
14
from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
15
SequenceGroupMetadata, SequenceStatus)
16
+from vllm.utils import PyObjectCache
17
18
logger = init_logger(__name__)
19
@@ -176,10 +177,10 @@ class SchedulerRunningOutputs:
177
enough memory, it can be preempted (for recompute) or swapped out.
178
"""
179
# Selected sequences that are running and in a decoding phase.
179
- decode_seq_groups: List[SequenceGroup]
180
+ decode_seq_groups: List[ScheduledSequenceGroup]
181
# Selected sequences that are running and in a prefill phase.
182
# I.e., it means the prefill has been chunked.
182
- prefill_seq_groups: List[SequenceGroup]
183
+ prefill_seq_groups: List[ScheduledSequenceGroup]
184
# The preempted sequences.
185
preempted: List[SequenceGroup]
186
# Sequences that are swapped out.
@@ -191,6 +192,10 @@ class SchedulerRunningOutputs:
192
# The number of slots for lookahead decoding.
193
num_lookahead_slots: int
194
195
+ # Optimization for fast-access to seq_group lists
196
+ decode_seq_groups_list: List[SequenceGroup]
197
+ prefill_seq_groups_list: List[SequenceGroup]
198
+
199
@classmethod
200
def create_empty(cls) -> "SchedulerRunningOutputs":
201
return SchedulerRunningOutputs(
@@ -201,6 +206,8 @@ class SchedulerRunningOutputs:
206
blocks_to_swap_out=[],
207
blocks_to_copy=[],
208
num_lookahead_slots=0,
209
+ decode_seq_groups_list=[],
210
+ prefill_seq_groups_list=[],
211
)
212
213
@@ -259,6 +266,30 @@ class SchedulerPrefillOutputs:
266
)
267
268
269
+def seq_group_metadata_builder():
270
+ return SequenceGroupMetadata(request_id="",
271
+ is_prompt=False,
272
+ seq_data={},
273
+ sampling_params=None,
274
+ block_tables={})
275
+
276
+
277
+def scheduler_running_outputs_builder():
278
+ return SchedulerRunningOutputs(decode_seq_groups=[],
279
+ prefill_seq_groups=[],
280
+ preempted=[],
281
+ swapped_out=[],
282
+ blocks_to_swap_out=[],
283
+ blocks_to_copy=[],
284
+ num_lookahead_slots=0,
285
+ prefill_seq_groups_list=[],
286
+ decode_seq_groups_list=[])
287
+
288
+
289
+def scheduled_seq_group_builder():
290
+ return ScheduledSequenceGroup(seq_group=None, token_chunk_size=0)
291
+
292
+
293
class Scheduler:
294
295
def __init__(
@@ -331,6 +362,14 @@ class Scheduler:
362
else 0)
363
self.num_cumulative_preemption: int = 0
364
365
+ # Used to cache python objects
366
+ self._seq_group_metadata_cache: PyObjectCache = PyObjectCache(
367
+ seq_group_metadata_builder)
368
+ self._scheduler_running_outputs_cache: PyObjectCache = PyObjectCache(
369
+ scheduler_running_outputs_builder)
370
+ self._scheduled_seq_group_cache: PyObjectCache = PyObjectCache(
371
+ scheduled_seq_group_builder)
372
+
373
@property
374
def lora_enabled(self) -> bool:
375
return bool(self.lora_config)
@@ -441,14 +480,30 @@ class Scheduler:
480
Returns:
481
SchedulerRunningOutputs.
482
"""
483
+ ret: SchedulerRunningOutputs = \
484
+ self._scheduler_running_outputs_cache.get_object()
485
+ ret.blocks_to_swap_out.clear()
486
+ ret.blocks_to_copy.clear()
487
+ ret.decode_seq_groups.clear()
488
+ ret.prefill_seq_groups.clear()
489
+ ret.preempted.clear()
490
+ ret.swapped_out.clear()
491
+
492
+ ret.num_lookahead_slots = self._get_num_lookahead_slots(
493
+ is_prefill=False)
494
+
495
+ ret.decode_seq_groups_list.clear()
496
+ ret.prefill_seq_groups_list.clear()
497
+
498
# Blocks that need to be swapped or copied before model execution.
445
- blocks_to_swap_out: List[Tuple[int, int]] = []
446
- blocks_to_copy: List[Tuple[int, int]] = []
499
+ blocks_to_swap_out: List[Tuple[int, int]] = ret.blocks_to_swap_out
500
+ blocks_to_copy: List[Tuple[int, int]] = ret.blocks_to_copy
501
448
- decode_seq_groups: List[ScheduledSequenceGroup] = []
449
- prefill_seq_groups: List[ScheduledSequenceGroup] = []
450
- preempted: List[SequenceGroup] = []
451
- swapped_out: List[SequenceGroup] = []
502
+ decode_seq_groups: List[ScheduledSequenceGroup] = ret.decode_seq_groups
503
+ prefill_seq_groups: List[
504
+ ScheduledSequenceGroup] = ret.prefill_seq_groups
505
+ preempted: List[SequenceGroup] = ret.preempted
506
+ swapped_out: List[SequenceGroup] = ret.swapped_out
507
508
# NOTE(woosuk): Preemption happens only when there is no available slot
509
# to keep all the sequence groups in the RUNNING state.
@@ -497,15 +552,19 @@ class Scheduler:
552
else:
553
self._append_slots(seq_group, blocks_to_copy)
554
is_prefill = seq_group.is_prefill()
555
+
556
+ scheduled_seq_group: ScheduledSequenceGroup = \
557
+ self._scheduled_seq_group_cache.get_object()
558
+ scheduled_seq_group.seq_group = seq_group
559
if is_prefill:
501
- prefill_seq_groups.append(
502
- ScheduledSequenceGroup(
503
- seq_group=seq_group,
504
- token_chunk_size=num_running_tokens))
560
+ scheduled_seq_group.token_chunk_size = num_running_tokens
561
+ prefill_seq_groups.append(scheduled_seq_group)
562
+ ret.prefill_seq_groups_list.append(seq_group)
563
else:
506
- decode_seq_groups.append(
507
- ScheduledSequenceGroup(seq_group=seq_group,
508
- token_chunk_size=1))
564
+ scheduled_seq_group.token_chunk_size = 1
565
+ decode_seq_groups.append(scheduled_seq_group)
566
+ ret.decode_seq_groups_list.append(seq_group)
567
+
568
budget.add_num_batched_tokens(seq_group.request_id,
569
num_running_tokens)
570
# OPTIMIZATION: Note that get_max_num_running_seqs is
@@ -518,15 +577,10 @@ class Scheduler:
577
if curr_loras is not None and seq_group.lora_int_id > 0:
578
curr_loras.add(seq_group.lora_int_id)
579
521
- return SchedulerRunningOutputs(
522
- decode_seq_groups=decode_seq_groups,
523
- prefill_seq_groups=prefill_seq_groups,
524
- preempted=preempted,
525
- swapped_out=swapped_out,
526
- blocks_to_swap_out=blocks_to_swap_out,
527
- blocks_to_copy=blocks_to_copy,
528
- num_lookahead_slots=self._get_num_lookahead_slots(
529
- is_prefill=False))
580
+ self._scheduler_running_outputs_cache.reset()
581
+ self._scheduled_seq_group_cache.reset()
582
+
583
+ return ret
584
585
def _schedule_swapped(
586
self,
@@ -820,11 +874,15 @@ class Scheduler:
874
# Update waiting requests.
875
self.waiting.extendleft(running_scheduled.preempted)
876
# Update new running requests.
823
- self.running.extend([s.seq_group for s in prefills.seq_groups])
824
- self.running.extend(
825
- [s.seq_group for s in running_scheduled.decode_seq_groups])
826
- self.running.extend(
827
- [s.seq_group for s in swapped_in.decode_seq_groups])
877
+ if len(prefills.seq_groups) > 0:
878
+ self.running.extend([s.seq_group for s in prefills.seq_groups])
879
+
880
+ self.running.extend(running_scheduled.decode_seq_groups_list)
881
+
882
+ if len(swapped_in.decode_seq_groups) > 0:
883
+ self.running.extend(
884
+ [s.seq_group for s in swapped_in.decode_seq_groups])
885
+
886
# Update swapped requests.
887
self.swapped.extend(running_scheduled.swapped_out)
888
preempted = (len(running_scheduled.preempted) +
@@ -834,18 +892,30 @@ class Scheduler:
892
# doesn't allow chunked prefills.
893
assert len(running_scheduled.prefill_seq_groups) == 0
894
assert len(swapped_in.prefill_seq_groups) == 0
895
+
896
+ # Merge lists
897
+ num_prefill_groups = len(prefills.seq_groups)
898
+ if num_prefill_groups > 0:
899
+ scheduled_seq_groups = prefills.seq_groups
900
+ scheduled_seq_groups.extend(running_scheduled.decode_seq_groups)
901
+ else:
902
+ scheduled_seq_groups = running_scheduled.decode_seq_groups
903
+ scheduled_seq_groups.extend(swapped_in.decode_seq_groups)
904
+
905
+ blocks_to_copy = running_scheduled.blocks_to_copy
906
+ blocks_to_copy.extend(swapped_in.blocks_to_copy)
907
+
908
+ ignored_seq_groups = prefills.ignored_seq_groups
909
+ ignored_seq_groups.extend(swapped_in.infeasible_seq_groups)
910
+
911
return SchedulerOutputs(
838
- scheduled_seq_groups=(prefills.seq_groups +
839
- running_scheduled.decode_seq_groups +
840
- swapped_in.decode_seq_groups),
841
- num_prefill_groups=len(prefills.seq_groups),
912
+ scheduled_seq_groups=scheduled_seq_groups,
913
+ num_prefill_groups=num_prefill_groups,
914
num_batched_tokens=budget.num_batched_tokens,
915
blocks_to_swap_in=swapped_in.blocks_to_swap_in,
916
blocks_to_swap_out=running_scheduled.blocks_to_swap_out,
845
- blocks_to_copy=running_scheduled.blocks_to_copy +
846
- swapped_in.blocks_to_copy,
847
- ignored_seq_groups=prefills.ignored_seq_groups +
848
- swapped_in.infeasible_seq_groups,
917
+ blocks_to_copy=blocks_to_copy,
918
+ ignored_seq_groups=ignored_seq_groups,
919
num_lookahead_slots=running_scheduled.num_lookahead_slots,
920
running_queue_size=len(self.running),
921
preempted=preempted,
@@ -963,6 +1033,9 @@ class Scheduler:
1033
scheduler_outputs = self._schedule()
1034
now = time.time()
1035
1036
+ if not self.cache_config.enable_prefix_caching:
1037
+ common_computed_block_nums = []
1038
+
1039
# Create input data structures.
1040
seq_group_metadata_list: List[SequenceGroupMetadata] = []
1041
for i, scheduled_seq_group in enumerate(
@@ -971,10 +1044,15 @@ class Scheduler:
1044
token_chunk_size = scheduled_seq_group.token_chunk_size
1045
seq_group.maybe_set_first_scheduled_time(now)
1046
1047
+ seq_group_metadata = self._seq_group_metadata_cache.get_object()
1048
+ seq_group_metadata.seq_data.clear()
1049
+ seq_group_metadata.block_tables.clear()
1050
+
1051
# seq_id -> SequenceData
975
- seq_data: Dict[int, SequenceData] = {}
1052
+ seq_data: Dict[int, SequenceData] = seq_group_metadata.seq_data
1053
# seq_id -> physical block numbers
977
- block_tables: Dict[int, List[int]] = {}
1054
+ block_tables: Dict[int,
1055
+ List[int]] = seq_group_metadata.block_tables
1056
1057
if seq_group.is_encoder_decoder():
1058
# Encoder associated with SequenceGroup
@@ -993,9 +1071,10 @@ class Scheduler:
1071
block_tables[seq_id] = self.block_manager.get_block_table(seq)
1072
self.block_manager.access_all_blocks_in_seq(seq, now)
1073
996
- common_computed_block_nums = (
997
- self.block_manager.get_common_computed_block_ids(
998
- seq_group.get_seqs(status=SequenceStatus.RUNNING)))
1074
+ if self.cache_config.enable_prefix_caching:
1075
+ common_computed_block_nums = (
1076
+ self.block_manager.get_common_computed_block_ids(
1077
+ seq_group.get_seqs(status=SequenceStatus.RUNNING)))
1078
1079
do_sample = True
1080
if seq_group.is_prefill():
@@ -1014,7 +1093,8 @@ class Scheduler:
1093
# It assumes the scheduled_seq_groups is ordered by
1094
# prefill < decoding.
1095
is_prompt = seq_group.is_prefill()
1017
- seq_group_metadata = SequenceGroupMetadata(
1096
+
1097
+ seq_group_metadata.__init__(
1098
request_id=seq_group.request_id,
1099
is_prompt=is_prompt,
1100
seq_data=seq_data,
@@ -1045,6 +1125,8 @@ class Scheduler:
1125
self.block_manager.mark_blocks_as_computed(
1126
scheduled_seq_group.seq_group)
1127
1128
+ self._seq_group_metadata_cache.reset()
1129
+
1130
return seq_group_metadata_list, scheduler_outputs
1131
1132
def fork_seq(self, parent_seq: Sequence, child_seq: Sequence) -> None:
@@ -1093,7 +1175,8 @@ class Scheduler:
1175
1176
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
1177
cows = self.block_manager.append_slots(seq, num_lookahead_slots)
1096
- blocks_to_copy.extend(cows)
1178
+ if len(cows) > 0:
1179
+ blocks_to_copy.extend(cows)
1180
1181
def _preempt(
1182
self,
vllm/model_executor/__init__.py
+3
-1
@@ -1,10 +1,12 @@
1
from vllm.model_executor.parameter import (BasevLLMParameter,
2
PackedvLLMParameter)
3
-from vllm.model_executor.sampling_metadata import SamplingMetadata
3
+from vllm.model_executor.sampling_metadata import (SamplingMetadata,
4
+ SamplingMetadataCache)
5
from vllm.model_executor.utils import set_random_seed
6
7
__all__ = [
8
"SamplingMetadata",
9
+ "SamplingMetadataCache",
10
"set_random_seed",
11
"BasevLLMParameter",
12
"PackedvLLMParameter",
vllm/model_executor/sampling_metadata.py
+71
-10
@@ -8,8 +8,9 @@ import torch
8
from vllm.sampling_params import SamplingParams, SamplingType
9
from vllm.sequence import SequenceData, SequenceGroupMetadata
10
from vllm.triton_utils.sample import get_num_triton_sampler_splits
11
-from vllm.utils import (async_tensor_h2d, is_pin_memory_available,
12
- make_tensor_with_pad, maybe_expand_dim)
11
+from vllm.utils import (PyObjectCache, async_tensor_h2d,
12
+ is_pin_memory_available, make_tensor_with_pad,
13
+ maybe_expand_dim)
14
15
_SAMPLING_EPS = 1e-5
16
_SEED_0_REPLACEMENT = 3403598558
@@ -62,6 +63,39 @@ class SequenceGroupToSample:
63
assert self.query_len is not None
64
65
66
+def gen_seq_group_to_sample_builder(num_seqs: int):
67
+ return lambda: SequenceGroupToSample(
68
+ seq_ids=[0] * num_seqs,
69
+ sampling_params=None,
70
+ seq_data=None, # type: ignore
71
+ seq_len=0,
72
+ query_len=0,
73
+ generator=None,
74
+ is_prompt=True,
75
+ prompt_logprob_indices=[],
76
+ sample_indices=[])
77
+
78
+
79
+class SamplingMetadataCache:
80
+ """Used to cache SamplingMetadata objects between scheduler iterations
81
+ """
82
+
83
+ def __init__(self):
84
+ self._seq_group_to_sample_cache: Dict[int, PyObjectCache] = {}
85
+
86
+ def get_cached_seq_group_to_sample(self, num_seqs):
87
+ if num_seqs not in self._seq_group_to_sample_cache:
88
+ self._seq_group_to_sample_cache[num_seqs] = PyObjectCache(
89
+ gen_seq_group_to_sample_builder(num_seqs))
90
+
91
+ obj = self._seq_group_to_sample_cache[num_seqs].get_object()
92
+ return obj
93
+
94
+ def reset(self):
95
+ for cache in self._seq_group_to_sample_cache.values():
96
+ cache.reset()
97
+
98
+
99
class SamplingMetadata:
100
"""Metadata for input sequences. Used in sampler.
101
@@ -121,6 +155,7 @@ class SamplingMetadata:
155
device: str,
156
pin_memory: bool,
157
generators: Optional[Dict[str, torch.Generator]] = None,
158
+ cache: Optional[SamplingMetadataCache] = None,
159
) -> "SamplingMetadata":
160
(
161
seq_groups,
@@ -128,7 +163,7 @@ class SamplingMetadata:
163
categorized_sample_indices,
164
num_prompts,
165
) = _prepare_seq_groups(seq_group_metadata_list, seq_lens, query_lens,
131
- device, generators)
166
+ device, generators, cache)
167
selected_token_indices = async_tensor_h2d(selected_token_indices,
168
dtype=torch.long,
169
target_device=device,
@@ -164,6 +199,7 @@ def _prepare_seq_groups(
199
query_lens: Optional[List[int]],
200
device: str,
201
generators: Optional[Dict[str, torch.Generator]] = None,
202
+ cache: Optional[SamplingMetadataCache] = None,
203
) -> Tuple[List[SequenceGroupToSample], List[int], Dict[
204
SamplingType, List[Tuple[int, int]]], int]:
205
"""Prepare sequence groups and indices for sampling.
@@ -210,15 +246,27 @@ def _prepare_seq_groups(
246
num_prompts = 0
247
248
for i, seq_group_metadata in enumerate(seq_group_metadata_list):
213
- seq_ids = list(seq_group_metadata.seq_data.keys())
249
+ seq_ids = seq_group_metadata.seq_data.keys()
250
+
251
+ if cache is not None:
252
+ sample_obj = cache.get_cached_seq_group_to_sample(len(seq_ids))
253
+
254
+ for j, seq_id in enumerate(seq_ids):
255
+ sample_obj.seq_ids[j] = seq_id
256
+
257
+ sample_obj.prompt_logprob_indices.clear()
258
+ sample_obj.sample_indices.clear()
259
+
260
sampling_params = seq_group_metadata.sampling_params
261
is_prompt = seq_group_metadata.is_prompt
262
generator: Optional[torch.Generator] = None
263
# If the current seq group is in decode stage, it is None.
264
seq_len: Optional[int] = None
265
query_len: Optional[int] = None
220
- prompt_logprob_indices: List[int] = []
221
- sample_indices: List[int] = []
266
+ prompt_logprob_indices: List[int] = \
267
+ sample_obj.prompt_logprob_indices if cache is not None else []
268
+ sample_indices: List[int] = \
269
+ sample_obj.sample_indices if cache is not None else []
270
do_sample = seq_group_metadata.do_sample
271
272
if seq_group_metadata.is_prompt:
@@ -290,9 +338,16 @@ def _prepare_seq_groups(
338
logit_idx += sample_len
339
sample_idx += sample_len
340
293
- seq_groups.append(
294
- SequenceGroupToSample(
295
- seq_ids=seq_ids,
341
+ if cache is not None:
342
+ sample_obj.sampling_params = sampling_params
343
+ sample_obj.seq_data = seq_group_metadata.seq_data
344
+ sample_obj.seq_len = seq_len
345
+ sample_obj.query_len = query_len
346
+ sample_obj.generator = generator
347
+ sample_obj.is_prompt = is_prompt
348
+ else:
349
+ sample_obj = SequenceGroupToSample(
350
+ seq_ids=list(seq_ids),
351
sampling_params=sampling_params,
352
seq_data=seq_group_metadata.seq_data,
353
seq_len=seq_len,
@@ -300,7 +355,13 @@ def _prepare_seq_groups(
355
generator=generator,
356
is_prompt=is_prompt,
357
prompt_logprob_indices=list(prompt_logprob_indices),
303
- sample_indices=list(sample_indices)))
358
+ sample_indices=list(sample_indices))
359
+
360
+ seq_groups.append(sample_obj)
361
+
362
+ if cache is not None:
363
+ cache.reset()
364
+
365
return (seq_groups, selected_token_indices, categorized_sample_indices,
366
num_prompts)
367
vllm/outputs.py
+1
-1
@@ -139,7 +139,7 @@ class RequestOutput:
139
CompletionOutput(
140
seqs.index(seq),
141
seq.get_output_text_to_return(text_buffer_length),
142
- seq.get_output_token_ids(),
142
+ seq.data._output_token_ids, # type: ignore
143
seq.get_cumulative_logprob() if include_logprobs else None,
144
seq.output_logprobs if include_logprobs else None,
145
SequenceStatus.get_finished_reason(seq.status),
vllm/sequence.py
+24
-4
@@ -1,7 +1,6 @@
1
"""Sequence and its related classes."""
2
import copy
3
import enum
4
-import math
4
from abc import ABC, abstractmethod
5
from array import array
6
from collections import defaultdict
@@ -330,7 +329,7 @@ class Sequence:
329
330
@property
331
def n_blocks(self) -> int:
333
- return math.ceil(self.get_len() / self.block_size)
332
+ return (self.get_len() + self.block_size - 1) // self.block_size
333
334
@property
335
def prompt(self) -> Optional[str]:
@@ -514,7 +513,9 @@ class SequenceGroup:
513
) -> None:
514
self.request_id = request_id
515
self.seqs = seqs
516
+ self.is_single_seq = len(seqs) == 1
517
self.seqs_dict = {seq.seq_id: seq for seq in seqs}
518
+
519
self.sampling_params = sampling_params
520
self.metrics = RequestMetrics(arrival_time=arrival_time,
521
last_token_time=arrival_time,
@@ -635,6 +636,10 @@ class SequenceGroup:
636
) -> List[Sequence]:
637
if status is None:
638
return self.seqs
639
+
640
+ if self.is_single_seq:
641
+ return self.seqs if self.seqs[0].status == status else []
642
+
643
return [seq for seq in self.seqs if seq.status == status]
644
645
def is_encoder_decoder(self) -> bool:
@@ -644,6 +649,9 @@ class SequenceGroup:
649
return self.encoder_seq
650
651
def get_unfinished_seqs(self) -> List[Sequence]:
652
+ if self.is_single_seq:
653
+ return self.seqs if not self.seqs[0].is_finished() else []
654
+
655
return [seq for seq in self.seqs if not seq.is_finished()]
656
657
def get_finished_seqs(self) -> List[Sequence]:
@@ -668,12 +676,21 @@ class SequenceGroup:
676
if status is None:
677
return len(self.seqs)
678
679
+ if self.is_single_seq:
680
+ return 1 if self.seqs[0].status == status else 0
681
+
682
return len(self.get_seqs(status))
683
684
def num_unfinished_seqs(self) -> int:
685
+ if self.is_single_seq:
686
+ return 1 if not self.seqs[0].is_finished() else 0
687
+
688
return len(self.get_unfinished_seqs())
689
690
def num_finished_seqs(self) -> int:
691
+ if self.is_single_seq:
692
+ return 1 if self.seqs[0].is_finished() else 0
693
+
694
return len(self.get_finished_seqs())
695
696
def find(self, seq_id: int) -> Sequence:
@@ -686,12 +703,14 @@ class SequenceGroup:
703
raise ValueError(f"Sequence {seq.seq_id} already exists.")
704
self.seqs_dict[seq.seq_id] = seq
705
self.seqs.append(seq)
706
+ self.is_single_seq = len(self.seqs) == 1
707
708
def remove(self, seq_id: int) -> None:
709
seq = self.seqs_dict.pop(seq_id, None)
710
if seq is None:
711
raise ValueError(f"Sequence {seq_id} not found.")
712
self.seqs.remove(seq)
713
+ self.is_single_seq = len(self.seqs) == 1
714
715
def is_finished(self) -> bool:
716
return all(seq.is_finished() for seq in self.seqs)
@@ -775,9 +794,10 @@ class SequenceGroupMetadata:
794
# TODO: We should maintain this states out of the sequence group.
795
self.num_speculative_tokens = None
796
778
- if self._token_chunk_size is None:
797
+ if seq_data is not None and self._token_chunk_size is None:
798
if is_prompt:
780
- self._token_chunk_size = list(seq_data.values())[0].get_len()
799
+ self._token_chunk_size = next(iter(
800
+ seq_data.values())).get_len()
801
else:
802
self._token_chunk_size = 1
803
vllm/utils.py
+38
@@ -261,6 +261,44 @@ class LRUCache(Generic[T]):
261
self.cache.clear()
262
263
264
+class PyObjectCache:
265
+ """Used to cache python objects to avoid object allocations
266
+ across scheduler iterations.
267
+ """
268
+
269
+ def __init__(self, obj_builder):
270
+ self._obj_builder = obj_builder
271
+ self._index = 0
272
+
273
+ self._obj_cache = []
274
+ for _ in range(128):
275
+ self._obj_cache.append(self._obj_builder())
276
+
277
+ def _grow_cache(self):
278
+ # Double the size of the cache
279
+ num_objs = len(self._obj_cache)
280
+ for _ in range(num_objs):
281
+ self._obj_cache.append(self._obj_builder())
282
+
283
+ def get_object(self):
284
+ """Returns a pre-allocated cached object. If there is not enough
285
+ objects, then the cache size will double.
286
+ """
287
+ if self._index >= len(self._obj_cache):
288
+ self._grow_cache()
289
+ assert self._index < len(self._obj_cache)
290
+
291
+ obj = self._obj_cache[self._index]
292
+ self._index += 1
293
+
294
+ return obj
295
+
296
+ def reset(self):
297
+ """Makes all cached-objects available for the next scheduler iteration.
298
+ """
299
+ self._index = 0
300
+
301
+
302
def is_hip() -> bool:
303
return torch.version.hip is not None
304
vllm/worker/model_runner.py
+211
-47
@@ -1,5 +1,6 @@
1
import dataclasses
2
import gc
3
+import itertools
4
import time
5
import warnings
6
import weakref
@@ -35,7 +36,7 @@ from vllm.logger import init_logger
36
from vllm.lora.layers import LoRAMapping
37
from vllm.lora.request import LoRARequest
38
from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
38
-from vllm.model_executor import SamplingMetadata
39
+from vllm.model_executor import SamplingMetadata, SamplingMetadataCache
40
from vllm.model_executor.model_loader import get_model
41
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
42
from vllm.model_executor.models.interfaces import (supports_lora,
@@ -50,8 +51,8 @@ from vllm.prompt_adapter.worker_manager import (
51
from vllm.sampling_params import SamplingParams
52
from vllm.sequence import (IntermediateTensors, SamplerOutput,
53
SequenceGroupMetadata)
53
-from vllm.utils import (CudaMemoryProfiler, async_tensor_h2d, flatten_2d_lists,
54
- get_kv_cache_torch_dtype, is_hip,
54
+from vllm.utils import (CudaMemoryProfiler, PyObjectCache, async_tensor_h2d,
55
+ flatten_2d_lists, get_kv_cache_torch_dtype, is_hip,
56
is_pin_memory_available)
57
from vllm.worker.model_runner_base import (
58
ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
@@ -178,6 +179,20 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
179
class InterDataForSeqGroup:
180
"""Intermediate data for the current sequence group."""
181
182
+ def simple_reinit(self):
183
+ self.input_tokens[0].clear() # type: ignore
184
+ self.input_positions[0].clear() # type: ignore
185
+ self.seq_lens[0] = 0 # type: ignore
186
+ self.orig_seq_lens[0] = 0 # type: ignore
187
+ self.query_lens[0] = 0 # type: ignore
188
+ self.context_lens[0] = 0 # type: ignore
189
+ self.curr_sliding_window_blocks[0] = 0 # type: ignore
190
+ self.lora_index_mapping.clear() # type: ignore
191
+ self.lora_prompt_mapping.clear() # type: ignore
192
+ self.lora_requests.clear() # type: ignore
193
+ self.prompt_adapter_index_mapping.clear() # type: ignore
194
+ self.prompt_adapter_prompt_mapping.clear() # type: ignore
195
+
196
def __init__(
197
self,
198
*,
@@ -220,35 +235,121 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
235
236
# Whether the prefix cache is hit (prefill only).
237
prefix_cache_hit: bool = False,
238
+ reinit: bool = False,
239
+ reinit_use_defaults: bool = False,
240
):
241
+ if reinit:
242
+ assert len(self.seq_ids) == len(seq_ids) # type: ignore
243
+ for i, seq_id in enumerate(seq_ids):
244
+ self.seq_ids[i] = seq_id # type: ignore
245
+ else:
246
+ self.seq_ids = seq_ids
247
+
248
self.request_id = request_id
225
- self.seq_ids = seq_ids
249
self.is_prompt = is_prompt
250
self.block_tables = block_tables
251
self.computed_block_nums = computed_block_nums
252
self.n_seqs = n_seqs
230
- self.input_tokens = input_tokens or []
231
- self.input_positions = input_positions or []
232
- self.seq_lens = seq_lens or []
233
- self.orig_seq_lens = orig_seq_lens or []
234
- self.query_lens = query_lens or []
235
- self.context_lens = context_lens or []
236
- self.curr_sliding_window_blocks = curr_sliding_window_blocks or []
237
-
238
- self.lora_index_mapping = lora_index_mapping or []
239
- self.lora_prompt_mapping = lora_prompt_mapping or []
240
- self.lora_requests = lora_requests or set()
241
-
242
- self.prompt_adapter_index_mapping = (prompt_adapter_index_mapping
243
- or [])
244
- self.prompt_adapter_prompt_mapping = (prompt_adapter_prompt_mapping
245
- or [])
246
- self.prompt_adapter_request = prompt_adapter_request
253
254
+ if reinit:
255
+ if len(self.seq_ids) == 1 and reinit_use_defaults:
256
+ self.simple_reinit()
257
+ else:
258
+ if input_tokens:
259
+ self.input_tokens = input_tokens
260
+ else:
261
+ for seq_id in range(len(self.seq_ids)):
262
+ self.input_tokens[seq_id].clear()
263
+
264
+ if input_positions:
265
+ self.input_positions = input_positions
266
+ else:
267
+ for seq_id in range(len(self.seq_ids)):
268
+ self.input_positions[seq_id].clear()
269
+
270
+ if seq_lens:
271
+ self.seq_lens = seq_lens
272
+ else:
273
+ for seq_id in range(len(self.seq_ids)):
274
+ self.seq_lens[seq_id] = 0
275
+
276
+ if orig_seq_lens:
277
+ self.orig_seq_lens = orig_seq_lens
278
+ else:
279
+ for seq_id in range(len(self.seq_ids)):
280
+ self.orig_seq_lens[seq_id] = 0
281
+
282
+ if query_lens:
283
+ self.query_lens = query_lens
284
+ else:
285
+ for seq_id in range(len(self.seq_ids)):
286
+ self.query_lens[seq_id] = 0
287
+
288
+ if context_lens:
289
+ self.context_lens = context_lens
290
+ else:
291
+ for seq_id in range(len(self.seq_ids)):
292
+ self.context_lens[seq_id] = 0
293
+
294
+ if curr_sliding_window_blocks:
295
+ self.curr_sliding_window_blocks = \
296
+ curr_sliding_window_blocks
297
+ else:
298
+ for seq_id in range(len(self.seq_ids)):
299
+ self.curr_sliding_window_blocks[seq_id] = 0
300
+
301
+ if lora_index_mapping:
302
+ self.lora_index_mapping = lora_index_mapping
303
+ else:
304
+ self.lora_index_mapping.clear()
305
+
306
+ if lora_prompt_mapping:
307
+ self.lora_prompt_mapping = lora_prompt_mapping
308
+ else:
309
+ self.lora_prompt_mapping.clear()
310
+
311
+ if lora_requests:
312
+ self.lora_requests = lora_requests
313
+ else:
314
+ self.lora_requests.clear()
315
+
316
+ if prompt_adapter_index_mapping:
317
+ self.prompt_adapter_index_mapping = \
318
+ prompt_adapter_index_mapping
319
+ else:
320
+ self.prompt_adapter_index_mapping.clear()
321
+
322
+ if prompt_adapter_prompt_mapping:
323
+ self.prompt_adapter_prompt_mapping = \
324
+ prompt_adapter_prompt_mapping
325
+ else:
326
+ self.prompt_adapter_prompt_mapping.clear()
327
+
328
+ else:
329
+ self.input_tokens = input_tokens or []
330
+ self.input_positions = input_positions or []
331
+ self.seq_lens = seq_lens or []
332
+ self.orig_seq_lens = orig_seq_lens or []
333
+ self.query_lens = query_lens or []
334
+ self.context_lens = context_lens or []
335
+ self.curr_sliding_window_blocks = \
336
+ curr_sliding_window_blocks or []
337
+
338
+ self.lora_index_mapping = lora_index_mapping or []
339
+ self.lora_prompt_mapping = lora_prompt_mapping or []
340
+ self.lora_requests = lora_requests or set()
341
+
342
+ self.prompt_adapter_index_mapping = (
343
+ prompt_adapter_index_mapping or [])
344
+ self.prompt_adapter_prompt_mapping = (
345
+ prompt_adapter_prompt_mapping or [])
346
+
347
+ self.prompt_adapter_request = prompt_adapter_request
348
self.multi_modal_inputs = multi_modal_inputs
349
self.prefix_cache_hit = prefix_cache_hit
350
251
- self.__post_init__()
351
+ if not reinit:
352
+ self.__post_init__()
353
354
def __post_init__(self):
355
self.n_seqs = len(self.seq_ids)
@@ -261,8 +362,36 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
362
self.context_lens = [0] * self.n_seqs
363
self.curr_sliding_window_blocks = [0] * self.n_seqs
364
264
- self.lora_index_mapping = [[] for _ in range(self.n_seqs)]
265
- self.lora_prompt_mapping = [[] for _ in range(self.n_seqs)]
365
+ self.lora_index_mapping = []
366
+ self.lora_prompt_mapping = []
367
+
368
+ def gen_inter_data_builder(self, num_seqs: int):
369
+ return lambda: ModelInputForGPUBuilder.InterDataForSeqGroup(
370
+ request_id="",
371
+ seq_ids=[0] * num_seqs,
372
+ is_prompt=True,
373
+ block_tables=None,
374
+ computed_block_nums=[])
375
+
376
+ def init_cached_inter_data(self, *args, **kwargs):
377
+ assert len(args) == 0
378
+ assert "seq_ids" in kwargs
379
+ seq_ids = kwargs["seq_ids"]
380
+ num_seqs = len(seq_ids)
381
+
382
+ # The inter-data cache is per model_runner
383
+ inter_data_cache = self.runner.inter_data_cache
384
+ if num_seqs not in inter_data_cache:
385
+ inter_data_cache[num_seqs] = PyObjectCache(
386
+ self.gen_inter_data_builder(num_seqs))
387
+
388
+ obj = inter_data_cache[num_seqs].get_object()
389
+ obj.__init__(*args, **kwargs)
390
+ return obj
391
+
392
+ def reset_cached_inter_data(self):
393
+ for cache in self.runner.inter_data_cache.values():
394
+ cache.reset()
395
396
def __init__(self,
397
runner: "GPUModelRunnerBase",
@@ -337,17 +466,29 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
466
467
# Compute tokens.
468
if inter_data.is_prompt:
340
- tokens = seq_data.get_token_ids()[context_len:seq_len]
469
+ tokens = seq_data.get_token_ids()
470
+ if context_len != 0 or seq_len < len(tokens):
471
+ tokens = tokens[context_len:seq_len]
472
else:
473
# Optimization. get_token_ids requires the entire copy of
474
# tokens.
344
- tokens = [seq_data.get_last_token_id()]
475
+ tokens = seq_data.get_last_token_id()
476
477
inter_data.seq_lens[seq_idx] = seq_len
478
inter_data.orig_seq_lens[seq_idx] = seq_len
479
inter_data.context_lens[seq_idx] = context_len
349
- inter_data.input_tokens[seq_idx] = tokens
350
- inter_data.input_positions[seq_idx] = list(range(context_len, seq_len))
480
+
481
+ if isinstance(tokens, list):
482
+ inter_data.input_tokens[seq_idx].extend(tokens)
483
+ else:
484
+ inter_data.input_tokens[seq_idx].append(tokens)
485
+
486
+ if (seq_len - context_len) == 1:
487
+ inter_data.input_positions[seq_idx].append(seq_len - 1)
488
+ else:
489
+ inter_data.input_positions[seq_idx].extend(
490
+ range(context_len, seq_len))
491
+
492
inter_data.query_lens[
493
seq_idx] = seq_len - context_len if inter_data.is_prompt else 1
494
@@ -471,7 +612,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
612
613
def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata):
614
"""Add a sequence group to the builder."""
474
- seq_ids = list(seq_group_metadata.seq_data.keys())
615
+ seq_ids = seq_group_metadata.seq_data.keys()
616
n_seqs = len(seq_ids)
617
is_prompt = seq_group_metadata.is_prompt
618
@@ -479,12 +620,15 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
620
assert n_seqs == 1
621
self.decode_only = False
622
482
- inter_data = self.InterDataForSeqGroup(
623
+ inter_data = self.init_cached_inter_data(
624
request_id=seq_group_metadata.request_id,
625
seq_ids=seq_ids,
626
is_prompt=is_prompt,
627
block_tables=seq_group_metadata.block_tables,
487
- computed_block_nums=seq_group_metadata.computed_block_nums)
628
+ computed_block_nums=seq_group_metadata.computed_block_nums,
629
+ reinit=True,
630
+ reinit_use_defaults=True)
631
+
632
self.inter_data_list.append(inter_data)
633
634
for seq_idx in range(n_seqs):
@@ -504,18 +648,21 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
648
create on-device tensors.
649
"""
650
# Combine and flatten intermediate data.
507
- input_tokens = flatten_2d_lists([
508
- flatten_2d_lists(inter_data.input_tokens)
509
- for inter_data in self.inter_data_list
510
- ])
651
+ input_tokens = []
652
+ for inter_data in self.inter_data_list:
653
+ for cur_input_tokens in inter_data.input_tokens:
654
+ input_tokens.extend(cur_input_tokens)
655
+
656
if not input_tokens:
657
# This may happen when all prefill requests hit
658
# prefix caching and there is no decode request.
659
return self.model_input_cls()
515
- input_positions = flatten_2d_lists([
516
- flatten_2d_lists(inter_data.input_positions)
517
- for inter_data in self.inter_data_list
518
- ])
660
+
661
+ input_positions = []
662
+ for inter_data in self.inter_data_list:
663
+ for cur_input_positions in inter_data.input_positions:
664
+ input_positions.extend(cur_input_positions)
665
+
666
seq_lens = []
667
max_decode_seq_len = 0
668
for inter_data in self.inter_data_list:
@@ -523,8 +670,10 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
670
if not inter_data.is_prompt:
671
max_decode_seq_len = max(max_decode_seq_len,
672
max(inter_data.seq_lens))
526
- query_lens = flatten_2d_lists(
527
- [inter_data.query_lens for inter_data in self.inter_data_list])
673
+ query_lens = []
674
+ for inter_data in self.inter_data_list:
675
+ query_lens.extend(inter_data.query_lens)
676
+
677
# Mapping from request IDs to sequence IDs. Used for Jamba models
678
# that manages the cache by itself.
679
request_ids_to_seq_ids = {
@@ -547,8 +696,9 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
696
batch_size = graph_batch_size
697
698
# Tokens and positions.
550
- input_tokens.extend([0] * cuda_graph_pad_size)
551
- input_positions.extend([0] * cuda_graph_pad_size)
699
+ if cuda_graph_pad_size:
700
+ input_tokens.extend(itertools.repeat(0, cuda_graph_pad_size))
701
+ input_positions.extend(itertools.repeat(0, cuda_graph_pad_size))
702
assert self.runner.device is not None
703
input_tokens_tensor = async_tensor_h2d(input_tokens, torch.long,
704
self.runner.device,
@@ -558,7 +708,8 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
708
self.runner.pin_memory)
709
710
# Sequence and query lengths.
561
- seq_lens.extend([1] * cuda_graph_pad_size)
711
+ if cuda_graph_pad_size:
712
+ seq_lens.extend(itertools.repeat(1, cuda_graph_pad_size))
713
714
# Attention metadata.
715
attn_metadata = self.attn_metadata_builder.build(
@@ -574,11 +725,14 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
725
flatten_2d_lists(inter_data.lora_index_mapping)
726
for inter_data in self.inter_data_list
727
])
577
- lora_index_mapping.extend([0] * cuda_graph_pad_size)
728
+ if cuda_graph_pad_size:
729
+ lora_index_mapping.extend(
730
+ itertools.repeat(0, cuda_graph_pad_size))
731
lora_prompt_mapping = flatten_2d_lists([
732
flatten_2d_lists(inter_data.lora_prompt_mapping)
733
for inter_data in self.inter_data_list
734
])
735
+
736
lora_mapping = LoRAMapping(
737
**dict(index_mapping=lora_index_mapping,
738
prompt_mapping=lora_prompt_mapping,
@@ -595,7 +749,9 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
749
inter_data.prompt_adapter_index_mapping
750
for inter_data in self.inter_data_list
751
])
598
- prompt_adapter_index_mapping.extend([0] * cuda_graph_pad_size)
752
+ if cuda_graph_pad_size:
753
+ prompt_adapter_index_mapping.extend(
754
+ itertools.repeat(0, cuda_graph_pad_size))
755
prompt_adapter_prompt_mapping = flatten_2d_lists([
756
inter_data.prompt_adapter_prompt_mapping
757
for inter_data in self.inter_data_list
@@ -717,6 +873,11 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
873
set_cpu_offload_max_bytes(
874
int(self.cache_config.cpu_offload_gb * 1024**3))
875
876
+ # Used to cache python objects
877
+ self.inter_data_cache: Dict[int, PyObjectCache] = {}
878
+ self.sampling_metadata_cache: SamplingMetadataCache = \
879
+ SamplingMetadataCache()
880
+
881
def load_model(self) -> None:
882
logger.info("Starting to load model %s...", self.model_config.model)
883
with CudaMemoryProfiler() as m:
@@ -843,6 +1004,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1004
builder = self._builder_cls(weakref.proxy(self), finished_requests_ids)
1005
for seq_group_metadata in seq_group_metadata_list:
1006
builder.add_seq_group(seq_group_metadata)
1007
+
1008
+ builder.reset_cached_inter_data()
1009
+
1010
return builder.build() # type: ignore
1011
1012
@torch.inference_mode()
@@ -1276,7 +1440,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1440
sampling_metadata = SamplingMetadata.prepare(
1441
seq_group_metadata_list, model_input.seq_lens,
1442
model_input.query_lens, self.device, self.pin_memory,
1279
- generators)
1443
+ generators, self.sampling_metadata_cache)
1444
else:
1445
sampling_metadata = None
1446
is_prompt = (seq_group_metadata_list[0].is_prompt