@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e02ac55617
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Performance] Optimize e2e overheads: Reduce python allocations (#7162)

Alexander Matveev committed Aug 9, 2024 at 00:34 UTC e02ac5561748306186aaeaad6dad4c89484a2b45
11 files changed +550 -125
vllm/attention/backends/flash_attn.py
+5 -1
@@ -259,7 +259,11 @@ class FlashAttentionMetadataBuilder(
259 block_table = block_tables[seq_id]
260 elif ((chunked_prefill_enabled or not is_prompt)
261 and block_tables is not None):
262 - block_table = block_tables[seq_id][-curr_sliding_window_block:]
262 + if curr_sliding_window_block == 0:
263 + block_table = block_tables[seq_id]
264 + else:
265 + block_table = block_tables[seq_id][
266 + -curr_sliding_window_block:]
267 self.block_tables.append(block_table)
268
269 # Compute slot mapping.
vllm/attention/backends/utils.py
+10 -2
@@ -68,13 +68,21 @@ def compute_slot_mapping(is_profile_run: bool, slot_mapping: List[int],
68 # tokens are masked and the slot mapping will be
69 # [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
70 block_table = block_tables[seq_id]
71 - slot_mapping.extend([PAD_SLOT_ID] * max(0, start_idx - context_len))
72 - for i in range(max(start_idx, context_len), seq_len):
71 +
72 + def add_slot(i):
73 block_number = block_table[i // block_size]
74 block_offset = i % block_size
75 slot = block_number * block_size + block_offset
76 slot_mapping.append(slot)
77
78 + if start_idx == 0 and (seq_len - context_len) == 1:
79 + # Optimization for common-case of decoding next token
80 + add_slot(seq_len - 1)
81 + else:
82 + slot_mapping.extend([PAD_SLOT_ID] * max(0, start_idx - context_len))
83 + for i in range(max(start_idx, context_len), seq_len):
84 + add_slot(i)
85 +
86
87 TAttentionMetadata = TypeVar("TAttentionMetadata", bound='AttentionMetadata')
88
vllm/block.py
+45 -3
@@ -1,5 +1,5 @@
1 """Token blocks."""
2 -from typing import List
2 +from typing import List, Optional
3
4 from vllm.utils import Device
5
@@ -37,5 +37,47 @@ class PhysicalTokenBlock:
37 f'computed={self.computed})')
38
39
40 -# Mapping: logical block number -> physical block.
41 -BlockTable = List[PhysicalTokenBlock]
40 +class BlockTable:
41 + """Holds a list of blocks with caching of their associated block_ids
42 + """
43 +
44 + def __init__(self, blocks: Optional[List[PhysicalTokenBlock]] = None):
45 + self._blocks: List[PhysicalTokenBlock] = []
46 + self._block_ids: List[int] = []
47 +
48 + if blocks is not None:
49 + for block in blocks:
50 + self.append(block)
51 +
52 + def append(self, block: PhysicalTokenBlock):
53 + self._blocks.append(block)
54 + self._block_ids.append(block.block_number)
55 +
56 + def __len__(self) -> int:
57 + return len(self._blocks)
58 +
59 + def __getitem__(self, key):
60 + return self._blocks[key]
61 +
62 + def __setitem__(self, key, value):
63 + if isinstance(key, slice):
64 + blocks = value
65 + self._blocks[key] = blocks
66 + self._block_ids[key] = [b.block_number for b in blocks]
67 + else:
68 + block = value
69 + self._blocks[key] = block
70 + self._block_ids[key] = block.block_number
71 +
72 + def reset(self):
73 + self._blocks = []
74 + self._block_ids = []
75 +
76 + def copy(self) -> "BlockTable":
77 + return BlockTable(self._blocks)
78 +
79 + def list(self) -> List[PhysicalTokenBlock]:
80 + return self._blocks
81 +
82 + def ids(self) -> List[int]:
83 + return self._block_ids
vllm/core/block_manager_v1.py
+15 -12
@@ -170,7 +170,7 @@ class UncachedBlockAllocator(BlockAllocatorBase):
170 self.num_blocks = num_blocks
171
172 # Initialize the free blocks.
173 - self.free_blocks: BlockTable = []
173 + self.free_blocks: List[PhysicalTokenBlock] = []
174 for i in range(num_blocks):
175 block = PhysicalTokenBlock(device=device,
176 block_number=i,
@@ -256,6 +256,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
256 Device.CPU, block_size, num_cpu_blocks)
257 # Mapping: seq_id -> BlockTable.
258 self.block_tables: Dict[int, BlockTable] = {}
259 +
260 # Mapping: req_id -> BlockTable
261 # Note that each SequenceGroup has a unique
262 # request ID
@@ -299,7 +300,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
300 # Allocate new physical token blocks that will store the prompt tokens.
301 num_prompt_blocks = seq.n_blocks
302
302 - block_table: BlockTable = []
303 + block_table: BlockTable = BlockTable()
304 for logical_idx in range(num_prompt_blocks):
305 if (self.block_sliding_window is not None
306 and logical_idx >= self.block_sliding_window):
@@ -326,15 +327,19 @@ class BlockSpaceManagerV1(BlockSpaceManager):
327 #
328 # NOTE: Here we assume that all sequences in the group have the same
329 # decoder prompt.
329 - seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
330 + wait_seqs = seq_group.get_seqs(status=SequenceStatus.WAITING)
331 + seq = wait_seqs[0]
332 block_table: BlockTable = \
333 self._allocate_sequence(seq,
334 seq_group.num_seqs(),
335 is_encoder_decoder)
336
337 # Assign the self-attention block tables for each sequence.
336 - for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
337 - self.block_tables[seq.seq_id] = block_table.copy()
338 + if len(wait_seqs) == 1:
339 + self.block_tables[wait_seqs[0].seq_id] = block_table
340 + else:
341 + for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
342 + self.block_tables[seq.seq_id] = block_table.copy()
343
344 # Allocate encoder sequence
345 if is_encoder_decoder:
@@ -476,6 +481,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
481 return
482 src_block_table = self.block_tables[parent_seq.seq_id]
483 self.block_tables[child_seq.seq_id] = src_block_table.copy()
484 +
485 # When using a sliding window, blocks will be eventually reused.
486 # In this case the block tables will contain repeated blocks.
487 # When forking, we must make sure that each block's `ref_count`
@@ -527,7 +533,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
533 dest_allocator: BlockAllocatorBase,
534 mapping: Dict[PhysicalTokenBlock,
535 PhysicalTokenBlock]) -> BlockTable:
530 - new_block_table = []
536 + new_block_table: BlockTable = BlockTable()
537
538 for from_block in block_table:
539 if from_block in mapping:
@@ -553,8 +559,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
559 for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
560 self.block_tables[seq.seq_id] = \
561 self._swap_block_table(self.block_tables[seq.seq_id],
556 - self.cpu_allocator,
557 - self.gpu_allocator,
562 + self.cpu_allocator, self.gpu_allocator,
563 mapping)
564
565 if seq_group.is_encoder_decoder():
@@ -580,8 +585,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
585 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
586 self.block_tables[seq.seq_id] = \
587 self._swap_block_table(self.block_tables[seq.seq_id],
583 - self.gpu_allocator,
584 - self.cpu_allocator,
588 + self.gpu_allocator, self.cpu_allocator,
589 mapping)
590
591 if seq_group.is_encoder_decoder():
@@ -636,8 +640,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
640 self.cross_block_tables.clear()
641
642 def get_block_table(self, seq: Sequence) -> List[int]:
639 - block_table = self.block_tables[seq.seq_id]
640 - return [block.block_number for block in block_table]
643 + return self.block_tables[seq.seq_id].ids()
644
645 def get_cross_block_table(self, seq_group: SequenceGroup) -> List[int]:
646 block_table = self.cross_block_tables[seq_group.request_id]
vllm/core/scheduler.py
+127 -44
@@ -13,6 +13,7 @@ from vllm.lora.request import LoRARequest
13 from vllm.prompt_adapter.request import PromptAdapterRequest
14 from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
15 SequenceGroupMetadata, SequenceStatus)
16 +from vllm.utils import PyObjectCache
17
18 logger = init_logger(__name__)
19
@@ -176,10 +177,10 @@ class SchedulerRunningOutputs:
177 enough memory, it can be preempted (for recompute) or swapped out.
178 """
179 # Selected sequences that are running and in a decoding phase.
179 - decode_seq_groups: List[SequenceGroup]
180 + decode_seq_groups: List[ScheduledSequenceGroup]
181 # Selected sequences that are running and in a prefill phase.
182 # I.e., it means the prefill has been chunked.
182 - prefill_seq_groups: List[SequenceGroup]
183 + prefill_seq_groups: List[ScheduledSequenceGroup]
184 # The preempted sequences.
185 preempted: List[SequenceGroup]
186 # Sequences that are swapped out.
@@ -191,6 +192,10 @@ class SchedulerRunningOutputs:
192 # The number of slots for lookahead decoding.
193 num_lookahead_slots: int
194
195 + # Optimization for fast-access to seq_group lists
196 + decode_seq_groups_list: List[SequenceGroup]
197 + prefill_seq_groups_list: List[SequenceGroup]
198 +
199 @classmethod
200 def create_empty(cls) -> "SchedulerRunningOutputs":
201 return SchedulerRunningOutputs(
@@ -201,6 +206,8 @@ class SchedulerRunningOutputs:
206 blocks_to_swap_out=[],
207 blocks_to_copy=[],
208 num_lookahead_slots=0,
209 + decode_seq_groups_list=[],
210 + prefill_seq_groups_list=[],
211 )
212
213
@@ -259,6 +266,30 @@ class SchedulerPrefillOutputs:
266 )
267
268
269 +def seq_group_metadata_builder():
270 + return SequenceGroupMetadata(request_id="",
271 + is_prompt=False,
272 + seq_data={},
273 + sampling_params=None,
274 + block_tables={})
275 +
276 +
277 +def scheduler_running_outputs_builder():
278 + return SchedulerRunningOutputs(decode_seq_groups=[],
279 + prefill_seq_groups=[],
280 + preempted=[],
281 + swapped_out=[],
282 + blocks_to_swap_out=[],
283 + blocks_to_copy=[],
284 + num_lookahead_slots=0,
285 + prefill_seq_groups_list=[],
286 + decode_seq_groups_list=[])
287 +
288 +
289 +def scheduled_seq_group_builder():
290 + return ScheduledSequenceGroup(seq_group=None, token_chunk_size=0)
291 +
292 +
293 class Scheduler:
294
295 def __init__(
@@ -331,6 +362,14 @@ class Scheduler:
362 else 0)
363 self.num_cumulative_preemption: int = 0
364
365 + # Used to cache python objects
366 + self._seq_group_metadata_cache: PyObjectCache = PyObjectCache(
367 + seq_group_metadata_builder)
368 + self._scheduler_running_outputs_cache: PyObjectCache = PyObjectCache(
369 + scheduler_running_outputs_builder)
370 + self._scheduled_seq_group_cache: PyObjectCache = PyObjectCache(
371 + scheduled_seq_group_builder)
372 +
373 @property
374 def lora_enabled(self) -> bool:
375 return bool(self.lora_config)
@@ -441,14 +480,30 @@ class Scheduler:
480 Returns:
481 SchedulerRunningOutputs.
482 """
483 + ret: SchedulerRunningOutputs = \
484 + self._scheduler_running_outputs_cache.get_object()
485 + ret.blocks_to_swap_out.clear()
486 + ret.blocks_to_copy.clear()
487 + ret.decode_seq_groups.clear()
488 + ret.prefill_seq_groups.clear()
489 + ret.preempted.clear()
490 + ret.swapped_out.clear()
491 +
492 + ret.num_lookahead_slots = self._get_num_lookahead_slots(
493 + is_prefill=False)
494 +
495 + ret.decode_seq_groups_list.clear()
496 + ret.prefill_seq_groups_list.clear()
497 +
498 # Blocks that need to be swapped or copied before model execution.
445 - blocks_to_swap_out: List[Tuple[int, int]] = []
446 - blocks_to_copy: List[Tuple[int, int]] = []
499 + blocks_to_swap_out: List[Tuple[int, int]] = ret.blocks_to_swap_out
500 + blocks_to_copy: List[Tuple[int, int]] = ret.blocks_to_copy
501
448 - decode_seq_groups: List[ScheduledSequenceGroup] = []
449 - prefill_seq_groups: List[ScheduledSequenceGroup] = []
450 - preempted: List[SequenceGroup] = []
451 - swapped_out: List[SequenceGroup] = []
502 + decode_seq_groups: List[ScheduledSequenceGroup] = ret.decode_seq_groups
503 + prefill_seq_groups: List[
504 + ScheduledSequenceGroup] = ret.prefill_seq_groups
505 + preempted: List[SequenceGroup] = ret.preempted
506 + swapped_out: List[SequenceGroup] = ret.swapped_out
507
508 # NOTE(woosuk): Preemption happens only when there is no available slot
509 # to keep all the sequence groups in the RUNNING state.
@@ -497,15 +552,19 @@ class Scheduler:
552 else:
553 self._append_slots(seq_group, blocks_to_copy)
554 is_prefill = seq_group.is_prefill()
555 +
556 + scheduled_seq_group: ScheduledSequenceGroup = \
557 + self._scheduled_seq_group_cache.get_object()
558 + scheduled_seq_group.seq_group = seq_group
559 if is_prefill:
501 - prefill_seq_groups.append(
502 - ScheduledSequenceGroup(
503 - seq_group=seq_group,
504 - token_chunk_size=num_running_tokens))
560 + scheduled_seq_group.token_chunk_size = num_running_tokens
561 + prefill_seq_groups.append(scheduled_seq_group)
562 + ret.prefill_seq_groups_list.append(seq_group)
563 else:
506 - decode_seq_groups.append(
507 - ScheduledSequenceGroup(seq_group=seq_group,
508 - token_chunk_size=1))
564 + scheduled_seq_group.token_chunk_size = 1
565 + decode_seq_groups.append(scheduled_seq_group)
566 + ret.decode_seq_groups_list.append(seq_group)
567 +
568 budget.add_num_batched_tokens(seq_group.request_id,
569 num_running_tokens)
570 # OPTIMIZATION: Note that get_max_num_running_seqs is
@@ -518,15 +577,10 @@ class Scheduler:
577 if curr_loras is not None and seq_group.lora_int_id > 0:
578 curr_loras.add(seq_group.lora_int_id)
579
521 - return SchedulerRunningOutputs(
522 - decode_seq_groups=decode_seq_groups,
523 - prefill_seq_groups=prefill_seq_groups,
524 - preempted=preempted,
525 - swapped_out=swapped_out,
526 - blocks_to_swap_out=blocks_to_swap_out,
527 - blocks_to_copy=blocks_to_copy,
528 - num_lookahead_slots=self._get_num_lookahead_slots(
529 - is_prefill=False))
580 + self._scheduler_running_outputs_cache.reset()
581 + self._scheduled_seq_group_cache.reset()
582 +
583 + return ret
584
585 def _schedule_swapped(
586 self,
@@ -820,11 +874,15 @@ class Scheduler:
874 # Update waiting requests.
875 self.waiting.extendleft(running_scheduled.preempted)
876 # Update new running requests.
823 - self.running.extend([s.seq_group for s in prefills.seq_groups])
824 - self.running.extend(
825 - [s.seq_group for s in running_scheduled.decode_seq_groups])
826 - self.running.extend(
827 - [s.seq_group for s in swapped_in.decode_seq_groups])
877 + if len(prefills.seq_groups) > 0:
878 + self.running.extend([s.seq_group for s in prefills.seq_groups])
879 +
880 + self.running.extend(running_scheduled.decode_seq_groups_list)
881 +
882 + if len(swapped_in.decode_seq_groups) > 0:
883 + self.running.extend(
884 + [s.seq_group for s in swapped_in.decode_seq_groups])
885 +
886 # Update swapped requests.
887 self.swapped.extend(running_scheduled.swapped_out)
888 preempted = (len(running_scheduled.preempted) +
@@ -834,18 +892,30 @@ class Scheduler:
892 # doesn't allow chunked prefills.
893 assert len(running_scheduled.prefill_seq_groups) == 0
894 assert len(swapped_in.prefill_seq_groups) == 0
895 +
896 + # Merge lists
897 + num_prefill_groups = len(prefills.seq_groups)
898 + if num_prefill_groups > 0:
899 + scheduled_seq_groups = prefills.seq_groups
900 + scheduled_seq_groups.extend(running_scheduled.decode_seq_groups)
901 + else:
902 + scheduled_seq_groups = running_scheduled.decode_seq_groups
903 + scheduled_seq_groups.extend(swapped_in.decode_seq_groups)
904 +
905 + blocks_to_copy = running_scheduled.blocks_to_copy
906 + blocks_to_copy.extend(swapped_in.blocks_to_copy)
907 +
908 + ignored_seq_groups = prefills.ignored_seq_groups
909 + ignored_seq_groups.extend(swapped_in.infeasible_seq_groups)
910 +
911 return SchedulerOutputs(
838 - scheduled_seq_groups=(prefills.seq_groups +
839 - running_scheduled.decode_seq_groups +
840 - swapped_in.decode_seq_groups),
841 - num_prefill_groups=len(prefills.seq_groups),
912 + scheduled_seq_groups=scheduled_seq_groups,
913 + num_prefill_groups=num_prefill_groups,
914 num_batched_tokens=budget.num_batched_tokens,
915 blocks_to_swap_in=swapped_in.blocks_to_swap_in,
916 blocks_to_swap_out=running_scheduled.blocks_to_swap_out,
845 - blocks_to_copy=running_scheduled.blocks_to_copy +
846 - swapped_in.blocks_to_copy,
847 - ignored_seq_groups=prefills.ignored_seq_groups +
848 - swapped_in.infeasible_seq_groups,
917 + blocks_to_copy=blocks_to_copy,
918 + ignored_seq_groups=ignored_seq_groups,
919 num_lookahead_slots=running_scheduled.num_lookahead_slots,
920 running_queue_size=len(self.running),
921 preempted=preempted,
@@ -963,6 +1033,9 @@ class Scheduler:
1033 scheduler_outputs = self._schedule()
1034 now = time.time()
1035
1036 + if not self.cache_config.enable_prefix_caching:
1037 + common_computed_block_nums = []
1038 +
1039 # Create input data structures.
1040 seq_group_metadata_list: List[SequenceGroupMetadata] = []
1041 for i, scheduled_seq_group in enumerate(
@@ -971,10 +1044,15 @@ class Scheduler:
1044 token_chunk_size = scheduled_seq_group.token_chunk_size
1045 seq_group.maybe_set_first_scheduled_time(now)
1046
1047 + seq_group_metadata = self._seq_group_metadata_cache.get_object()
1048 + seq_group_metadata.seq_data.clear()
1049 + seq_group_metadata.block_tables.clear()
1050 +
1051 # seq_id -> SequenceData
975 - seq_data: Dict[int, SequenceData] = {}
1052 + seq_data: Dict[int, SequenceData] = seq_group_metadata.seq_data
1053 # seq_id -> physical block numbers
977 - block_tables: Dict[int, List[int]] = {}
1054 + block_tables: Dict[int,
1055 + List[int]] = seq_group_metadata.block_tables
1056
1057 if seq_group.is_encoder_decoder():
1058 # Encoder associated with SequenceGroup
@@ -993,9 +1071,10 @@ class Scheduler:
1071 block_tables[seq_id] = self.block_manager.get_block_table(seq)
1072 self.block_manager.access_all_blocks_in_seq(seq, now)
1073
996 - common_computed_block_nums = (
997 - self.block_manager.get_common_computed_block_ids(
998 - seq_group.get_seqs(status=SequenceStatus.RUNNING)))
1074 + if self.cache_config.enable_prefix_caching:
1075 + common_computed_block_nums = (
1076 + self.block_manager.get_common_computed_block_ids(
1077 + seq_group.get_seqs(status=SequenceStatus.RUNNING)))
1078
1079 do_sample = True
1080 if seq_group.is_prefill():
@@ -1014,7 +1093,8 @@ class Scheduler:
1093 # It assumes the scheduled_seq_groups is ordered by
1094 # prefill < decoding.
1095 is_prompt = seq_group.is_prefill()
1017 - seq_group_metadata = SequenceGroupMetadata(
1096 +
1097 + seq_group_metadata.__init__(
1098 request_id=seq_group.request_id,
1099 is_prompt=is_prompt,
1100 seq_data=seq_data,
@@ -1045,6 +1125,8 @@ class Scheduler:
1125 self.block_manager.mark_blocks_as_computed(
1126 scheduled_seq_group.seq_group)
1127
1128 + self._seq_group_metadata_cache.reset()
1129 +
1130 return seq_group_metadata_list, scheduler_outputs
1131
1132 def fork_seq(self, parent_seq: Sequence, child_seq: Sequence) -> None:
@@ -1093,7 +1175,8 @@ class Scheduler:
1175
1176 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
1177 cows = self.block_manager.append_slots(seq, num_lookahead_slots)
1096 - blocks_to_copy.extend(cows)
1178 + if len(cows) > 0:
1179 + blocks_to_copy.extend(cows)
1180
1181 def _preempt(
1182 self,
vllm/model_executor/__init__.py
+3 -1
@@ -1,10 +1,12 @@
1 from vllm.model_executor.parameter import (BasevLLMParameter,
2 PackedvLLMParameter)
3 -from vllm.model_executor.sampling_metadata import SamplingMetadata
3 +from vllm.model_executor.sampling_metadata import (SamplingMetadata,
4 + SamplingMetadataCache)
5 from vllm.model_executor.utils import set_random_seed
6
7 __all__ = [
8 "SamplingMetadata",
9 + "SamplingMetadataCache",
10 "set_random_seed",
11 "BasevLLMParameter",
12 "PackedvLLMParameter",
vllm/model_executor/sampling_metadata.py
+71 -10
@@ -8,8 +8,9 @@ import torch
8 from vllm.sampling_params import SamplingParams, SamplingType
9 from vllm.sequence import SequenceData, SequenceGroupMetadata
10 from vllm.triton_utils.sample import get_num_triton_sampler_splits
11 -from vllm.utils import (async_tensor_h2d, is_pin_memory_available,
12 - make_tensor_with_pad, maybe_expand_dim)
11 +from vllm.utils import (PyObjectCache, async_tensor_h2d,
12 + is_pin_memory_available, make_tensor_with_pad,
13 + maybe_expand_dim)
14
15 _SAMPLING_EPS = 1e-5
16 _SEED_0_REPLACEMENT = 3403598558
@@ -62,6 +63,39 @@ class SequenceGroupToSample:
63 assert self.query_len is not None
64
65
66 +def gen_seq_group_to_sample_builder(num_seqs: int):
67 + return lambda: SequenceGroupToSample(
68 + seq_ids=[0] * num_seqs,
69 + sampling_params=None,
70 + seq_data=None, # type: ignore
71 + seq_len=0,
72 + query_len=0,
73 + generator=None,
74 + is_prompt=True,
75 + prompt_logprob_indices=[],
76 + sample_indices=[])
77 +
78 +
79 +class SamplingMetadataCache:
80 + """Used to cache SamplingMetadata objects between scheduler iterations
81 + """
82 +
83 + def __init__(self):
84 + self._seq_group_to_sample_cache: Dict[int, PyObjectCache] = {}
85 +
86 + def get_cached_seq_group_to_sample(self, num_seqs):
87 + if num_seqs not in self._seq_group_to_sample_cache:
88 + self._seq_group_to_sample_cache[num_seqs] = PyObjectCache(
89 + gen_seq_group_to_sample_builder(num_seqs))
90 +
91 + obj = self._seq_group_to_sample_cache[num_seqs].get_object()
92 + return obj
93 +
94 + def reset(self):
95 + for cache in self._seq_group_to_sample_cache.values():
96 + cache.reset()
97 +
98 +
99 class SamplingMetadata:
100 """Metadata for input sequences. Used in sampler.
101
@@ -121,6 +155,7 @@ class SamplingMetadata:
155 device: str,
156 pin_memory: bool,
157 generators: Optional[Dict[str, torch.Generator]] = None,
158 + cache: Optional[SamplingMetadataCache] = None,
159 ) -> "SamplingMetadata":
160 (
161 seq_groups,
@@ -128,7 +163,7 @@ class SamplingMetadata:
163 categorized_sample_indices,
164 num_prompts,
165 ) = _prepare_seq_groups(seq_group_metadata_list, seq_lens, query_lens,
131 - device, generators)
166 + device, generators, cache)
167 selected_token_indices = async_tensor_h2d(selected_token_indices,
168 dtype=torch.long,
169 target_device=device,
@@ -164,6 +199,7 @@ def _prepare_seq_groups(
199 query_lens: Optional[List[int]],
200 device: str,
201 generators: Optional[Dict[str, torch.Generator]] = None,
202 + cache: Optional[SamplingMetadataCache] = None,
203 ) -> Tuple[List[SequenceGroupToSample], List[int], Dict[
204 SamplingType, List[Tuple[int, int]]], int]:
205 """Prepare sequence groups and indices for sampling.
@@ -210,15 +246,27 @@ def _prepare_seq_groups(
246 num_prompts = 0
247
248 for i, seq_group_metadata in enumerate(seq_group_metadata_list):
213 - seq_ids = list(seq_group_metadata.seq_data.keys())
249 + seq_ids = seq_group_metadata.seq_data.keys()
250 +
251 + if cache is not None:
252 + sample_obj = cache.get_cached_seq_group_to_sample(len(seq_ids))
253 +
254 + for j, seq_id in enumerate(seq_ids):
255 + sample_obj.seq_ids[j] = seq_id
256 +
257 + sample_obj.prompt_logprob_indices.clear()
258 + sample_obj.sample_indices.clear()
259 +
260 sampling_params = seq_group_metadata.sampling_params
261 is_prompt = seq_group_metadata.is_prompt
262 generator: Optional[torch.Generator] = None
263 # If the current seq group is in decode stage, it is None.
264 seq_len: Optional[int] = None
265 query_len: Optional[int] = None
220 - prompt_logprob_indices: List[int] = []
221 - sample_indices: List[int] = []
266 + prompt_logprob_indices: List[int] = \
267 + sample_obj.prompt_logprob_indices if cache is not None else []
268 + sample_indices: List[int] = \
269 + sample_obj.sample_indices if cache is not None else []
270 do_sample = seq_group_metadata.do_sample
271
272 if seq_group_metadata.is_prompt:
@@ -290,9 +338,16 @@ def _prepare_seq_groups(
338 logit_idx += sample_len
339 sample_idx += sample_len
340
293 - seq_groups.append(
294 - SequenceGroupToSample(
295 - seq_ids=seq_ids,
341 + if cache is not None:
342 + sample_obj.sampling_params = sampling_params
343 + sample_obj.seq_data = seq_group_metadata.seq_data
344 + sample_obj.seq_len = seq_len
345 + sample_obj.query_len = query_len
346 + sample_obj.generator = generator
347 + sample_obj.is_prompt = is_prompt
348 + else:
349 + sample_obj = SequenceGroupToSample(
350 + seq_ids=list(seq_ids),
351 sampling_params=sampling_params,
352 seq_data=seq_group_metadata.seq_data,
353 seq_len=seq_len,
@@ -300,7 +355,13 @@ def _prepare_seq_groups(
355 generator=generator,
356 is_prompt=is_prompt,
357 prompt_logprob_indices=list(prompt_logprob_indices),
303 - sample_indices=list(sample_indices)))
358 + sample_indices=list(sample_indices))
359 +
360 + seq_groups.append(sample_obj)
361 +
362 + if cache is not None:
363 + cache.reset()
364 +
365 return (seq_groups, selected_token_indices, categorized_sample_indices,
366 num_prompts)
367
vllm/outputs.py
+1 -1
@@ -139,7 +139,7 @@ class RequestOutput:
139 CompletionOutput(
140 seqs.index(seq),
141 seq.get_output_text_to_return(text_buffer_length),
142 - seq.get_output_token_ids(),
142 + seq.data._output_token_ids, # type: ignore
143 seq.get_cumulative_logprob() if include_logprobs else None,
144 seq.output_logprobs if include_logprobs else None,
145 SequenceStatus.get_finished_reason(seq.status),
vllm/sequence.py
+24 -4
@@ -1,7 +1,6 @@
1 """Sequence and its related classes."""
2 import copy
3 import enum
4 -import math
4 from abc import ABC, abstractmethod
5 from array import array
6 from collections import defaultdict
@@ -330,7 +329,7 @@ class Sequence:
329
330 @property
331 def n_blocks(self) -> int:
333 - return math.ceil(self.get_len() / self.block_size)
332 + return (self.get_len() + self.block_size - 1) // self.block_size
333
334 @property
335 def prompt(self) -> Optional[str]:
@@ -514,7 +513,9 @@ class SequenceGroup:
513 ) -> None:
514 self.request_id = request_id
515 self.seqs = seqs
516 + self.is_single_seq = len(seqs) == 1
517 self.seqs_dict = {seq.seq_id: seq for seq in seqs}
518 +
519 self.sampling_params = sampling_params
520 self.metrics = RequestMetrics(arrival_time=arrival_time,
521 last_token_time=arrival_time,
@@ -635,6 +636,10 @@ class SequenceGroup:
636 ) -> List[Sequence]:
637 if status is None:
638 return self.seqs
639 +
640 + if self.is_single_seq:
641 + return self.seqs if self.seqs[0].status == status else []
642 +
643 return [seq for seq in self.seqs if seq.status == status]
644
645 def is_encoder_decoder(self) -> bool:
@@ -644,6 +649,9 @@ class SequenceGroup:
649 return self.encoder_seq
650
651 def get_unfinished_seqs(self) -> List[Sequence]:
652 + if self.is_single_seq:
653 + return self.seqs if not self.seqs[0].is_finished() else []
654 +
655 return [seq for seq in self.seqs if not seq.is_finished()]
656
657 def get_finished_seqs(self) -> List[Sequence]:
@@ -668,12 +676,21 @@ class SequenceGroup:
676 if status is None:
677 return len(self.seqs)
678
679 + if self.is_single_seq:
680 + return 1 if self.seqs[0].status == status else 0
681 +
682 return len(self.get_seqs(status))
683
684 def num_unfinished_seqs(self) -> int:
685 + if self.is_single_seq:
686 + return 1 if not self.seqs[0].is_finished() else 0
687 +
688 return len(self.get_unfinished_seqs())
689
690 def num_finished_seqs(self) -> int:
691 + if self.is_single_seq:
692 + return 1 if self.seqs[0].is_finished() else 0
693 +
694 return len(self.get_finished_seqs())
695
696 def find(self, seq_id: int) -> Sequence:
@@ -686,12 +703,14 @@ class SequenceGroup:
703 raise ValueError(f"Sequence {seq.seq_id} already exists.")
704 self.seqs_dict[seq.seq_id] = seq
705 self.seqs.append(seq)
706 + self.is_single_seq = len(self.seqs) == 1
707
708 def remove(self, seq_id: int) -> None:
709 seq = self.seqs_dict.pop(seq_id, None)
710 if seq is None:
711 raise ValueError(f"Sequence {seq_id} not found.")
712 self.seqs.remove(seq)
713 + self.is_single_seq = len(self.seqs) == 1
714
715 def is_finished(self) -> bool:
716 return all(seq.is_finished() for seq in self.seqs)
@@ -775,9 +794,10 @@ class SequenceGroupMetadata:
794 # TODO: We should maintain this states out of the sequence group.
795 self.num_speculative_tokens = None
796
778 - if self._token_chunk_size is None:
797 + if seq_data is not None and self._token_chunk_size is None:
798 if is_prompt:
780 - self._token_chunk_size = list(seq_data.values())[0].get_len()
799 + self._token_chunk_size = next(iter(
800 + seq_data.values())).get_len()
801 else:
802 self._token_chunk_size = 1
803
vllm/utils.py
+38
@@ -261,6 +261,44 @@ class LRUCache(Generic[T]):
261 self.cache.clear()
262
263
264 +class PyObjectCache:
265 + """Used to cache python objects to avoid object allocations
266 + across scheduler iterations.
267 + """
268 +
269 + def __init__(self, obj_builder):
270 + self._obj_builder = obj_builder
271 + self._index = 0
272 +
273 + self._obj_cache = []
274 + for _ in range(128):
275 + self._obj_cache.append(self._obj_builder())
276 +
277 + def _grow_cache(self):
278 + # Double the size of the cache
279 + num_objs = len(self._obj_cache)
280 + for _ in range(num_objs):
281 + self._obj_cache.append(self._obj_builder())
282 +
283 + def get_object(self):
284 + """Returns a pre-allocated cached object. If there is not enough
285 + objects, then the cache size will double.
286 + """
287 + if self._index >= len(self._obj_cache):
288 + self._grow_cache()
289 + assert self._index < len(self._obj_cache)
290 +
291 + obj = self._obj_cache[self._index]
292 + self._index += 1
293 +
294 + return obj
295 +
296 + def reset(self):
297 + """Makes all cached-objects available for the next scheduler iteration.
298 + """
299 + self._index = 0
300 +
301 +
302 def is_hip() -> bool:
303 return torch.version.hip is not None
304
vllm/worker/model_runner.py
+211 -47
@@ -1,5 +1,6 @@
1 import dataclasses
2 import gc
3 +import itertools
4 import time
5 import warnings
6 import weakref
@@ -35,7 +36,7 @@ from vllm.logger import init_logger
36 from vllm.lora.layers import LoRAMapping
37 from vllm.lora.request import LoRARequest
38 from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
38 -from vllm.model_executor import SamplingMetadata
39 +from vllm.model_executor import SamplingMetadata, SamplingMetadataCache
40 from vllm.model_executor.model_loader import get_model
41 from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
42 from vllm.model_executor.models.interfaces import (supports_lora,
@@ -50,8 +51,8 @@ from vllm.prompt_adapter.worker_manager import (
51 from vllm.sampling_params import SamplingParams
52 from vllm.sequence import (IntermediateTensors, SamplerOutput,
53 SequenceGroupMetadata)
53 -from vllm.utils import (CudaMemoryProfiler, async_tensor_h2d, flatten_2d_lists,
54 - get_kv_cache_torch_dtype, is_hip,
54 +from vllm.utils import (CudaMemoryProfiler, PyObjectCache, async_tensor_h2d,
55 + flatten_2d_lists, get_kv_cache_torch_dtype, is_hip,
56 is_pin_memory_available)
57 from vllm.worker.model_runner_base import (
58 ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
@@ -178,6 +179,20 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
179 class InterDataForSeqGroup:
180 """Intermediate data for the current sequence group."""
181
182 + def simple_reinit(self):
183 + self.input_tokens[0].clear() # type: ignore
184 + self.input_positions[0].clear() # type: ignore
185 + self.seq_lens[0] = 0 # type: ignore
186 + self.orig_seq_lens[0] = 0 # type: ignore
187 + self.query_lens[0] = 0 # type: ignore
188 + self.context_lens[0] = 0 # type: ignore
189 + self.curr_sliding_window_blocks[0] = 0 # type: ignore
190 + self.lora_index_mapping.clear() # type: ignore
191 + self.lora_prompt_mapping.clear() # type: ignore
192 + self.lora_requests.clear() # type: ignore
193 + self.prompt_adapter_index_mapping.clear() # type: ignore
194 + self.prompt_adapter_prompt_mapping.clear() # type: ignore
195 +
196 def __init__(
197 self,
198 *,
@@ -220,35 +235,121 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
235
236 # Whether the prefix cache is hit (prefill only).
237 prefix_cache_hit: bool = False,
238 + reinit: bool = False,
239 + reinit_use_defaults: bool = False,
240 ):
241 + if reinit:
242 + assert len(self.seq_ids) == len(seq_ids) # type: ignore
243 + for i, seq_id in enumerate(seq_ids):
244 + self.seq_ids[i] = seq_id # type: ignore
245 + else:
246 + self.seq_ids = seq_ids
247 +
248 self.request_id = request_id
225 - self.seq_ids = seq_ids
249 self.is_prompt = is_prompt
250 self.block_tables = block_tables
251 self.computed_block_nums = computed_block_nums
252 self.n_seqs = n_seqs
230 - self.input_tokens = input_tokens or []
231 - self.input_positions = input_positions or []
232 - self.seq_lens = seq_lens or []
233 - self.orig_seq_lens = orig_seq_lens or []
234 - self.query_lens = query_lens or []
235 - self.context_lens = context_lens or []
236 - self.curr_sliding_window_blocks = curr_sliding_window_blocks or []
237 -
238 - self.lora_index_mapping = lora_index_mapping or []
239 - self.lora_prompt_mapping = lora_prompt_mapping or []
240 - self.lora_requests = lora_requests or set()
241 -
242 - self.prompt_adapter_index_mapping = (prompt_adapter_index_mapping
243 - or [])
244 - self.prompt_adapter_prompt_mapping = (prompt_adapter_prompt_mapping
245 - or [])
246 - self.prompt_adapter_request = prompt_adapter_request
253
254 + if reinit:
255 + if len(self.seq_ids) == 1 and reinit_use_defaults:
256 + self.simple_reinit()
257 + else:
258 + if input_tokens:
259 + self.input_tokens = input_tokens
260 + else:
261 + for seq_id in range(len(self.seq_ids)):
262 + self.input_tokens[seq_id].clear()
263 +
264 + if input_positions:
265 + self.input_positions = input_positions
266 + else:
267 + for seq_id in range(len(self.seq_ids)):
268 + self.input_positions[seq_id].clear()
269 +
270 + if seq_lens:
271 + self.seq_lens = seq_lens
272 + else:
273 + for seq_id in range(len(self.seq_ids)):
274 + self.seq_lens[seq_id] = 0
275 +
276 + if orig_seq_lens:
277 + self.orig_seq_lens = orig_seq_lens
278 + else:
279 + for seq_id in range(len(self.seq_ids)):
280 + self.orig_seq_lens[seq_id] = 0
281 +
282 + if query_lens:
283 + self.query_lens = query_lens
284 + else:
285 + for seq_id in range(len(self.seq_ids)):
286 + self.query_lens[seq_id] = 0
287 +
288 + if context_lens:
289 + self.context_lens = context_lens
290 + else:
291 + for seq_id in range(len(self.seq_ids)):
292 + self.context_lens[seq_id] = 0
293 +
294 + if curr_sliding_window_blocks:
295 + self.curr_sliding_window_blocks = \
296 + curr_sliding_window_blocks
297 + else:
298 + for seq_id in range(len(self.seq_ids)):
299 + self.curr_sliding_window_blocks[seq_id] = 0
300 +
301 + if lora_index_mapping:
302 + self.lora_index_mapping = lora_index_mapping
303 + else:
304 + self.lora_index_mapping.clear()
305 +
306 + if lora_prompt_mapping:
307 + self.lora_prompt_mapping = lora_prompt_mapping
308 + else:
309 + self.lora_prompt_mapping.clear()
310 +
311 + if lora_requests:
312 + self.lora_requests = lora_requests
313 + else:
314 + self.lora_requests.clear()
315 +
316 + if prompt_adapter_index_mapping:
317 + self.prompt_adapter_index_mapping = \
318 + prompt_adapter_index_mapping
319 + else:
320 + self.prompt_adapter_index_mapping.clear()
321 +
322 + if prompt_adapter_prompt_mapping:
323 + self.prompt_adapter_prompt_mapping = \
324 + prompt_adapter_prompt_mapping
325 + else:
326 + self.prompt_adapter_prompt_mapping.clear()
327 +
328 + else:
329 + self.input_tokens = input_tokens or []
330 + self.input_positions = input_positions or []
331 + self.seq_lens = seq_lens or []
332 + self.orig_seq_lens = orig_seq_lens or []
333 + self.query_lens = query_lens or []
334 + self.context_lens = context_lens or []
335 + self.curr_sliding_window_blocks = \
336 + curr_sliding_window_blocks or []
337 +
338 + self.lora_index_mapping = lora_index_mapping or []
339 + self.lora_prompt_mapping = lora_prompt_mapping or []
340 + self.lora_requests = lora_requests or set()
341 +
342 + self.prompt_adapter_index_mapping = (
343 + prompt_adapter_index_mapping or [])
344 + self.prompt_adapter_prompt_mapping = (
345 + prompt_adapter_prompt_mapping or [])
346 +
347 + self.prompt_adapter_request = prompt_adapter_request
348 self.multi_modal_inputs = multi_modal_inputs
349 self.prefix_cache_hit = prefix_cache_hit
350
251 - self.__post_init__()
351 + if not reinit:
352 + self.__post_init__()
353
354 def __post_init__(self):
355 self.n_seqs = len(self.seq_ids)
@@ -261,8 +362,36 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
362 self.context_lens = [0] * self.n_seqs
363 self.curr_sliding_window_blocks = [0] * self.n_seqs
364
264 - self.lora_index_mapping = [[] for _ in range(self.n_seqs)]
265 - self.lora_prompt_mapping = [[] for _ in range(self.n_seqs)]
365 + self.lora_index_mapping = []
366 + self.lora_prompt_mapping = []
367 +
368 + def gen_inter_data_builder(self, num_seqs: int):
369 + return lambda: ModelInputForGPUBuilder.InterDataForSeqGroup(
370 + request_id="",
371 + seq_ids=[0] * num_seqs,
372 + is_prompt=True,
373 + block_tables=None,
374 + computed_block_nums=[])
375 +
376 + def init_cached_inter_data(self, *args, **kwargs):
377 + assert len(args) == 0
378 + assert "seq_ids" in kwargs
379 + seq_ids = kwargs["seq_ids"]
380 + num_seqs = len(seq_ids)
381 +
382 + # The inter-data cache is per model_runner
383 + inter_data_cache = self.runner.inter_data_cache
384 + if num_seqs not in inter_data_cache:
385 + inter_data_cache[num_seqs] = PyObjectCache(
386 + self.gen_inter_data_builder(num_seqs))
387 +
388 + obj = inter_data_cache[num_seqs].get_object()
389 + obj.__init__(*args, **kwargs)
390 + return obj
391 +
392 + def reset_cached_inter_data(self):
393 + for cache in self.runner.inter_data_cache.values():
394 + cache.reset()
395
396 def __init__(self,
397 runner: "GPUModelRunnerBase",
@@ -337,17 +466,29 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
466
467 # Compute tokens.
468 if inter_data.is_prompt:
340 - tokens = seq_data.get_token_ids()[context_len:seq_len]
469 + tokens = seq_data.get_token_ids()
470 + if context_len != 0 or seq_len < len(tokens):
471 + tokens = tokens[context_len:seq_len]
472 else:
473 # Optimization. get_token_ids requires the entire copy of
474 # tokens.
344 - tokens = [seq_data.get_last_token_id()]
475 + tokens = seq_data.get_last_token_id()
476
477 inter_data.seq_lens[seq_idx] = seq_len
478 inter_data.orig_seq_lens[seq_idx] = seq_len
479 inter_data.context_lens[seq_idx] = context_len
349 - inter_data.input_tokens[seq_idx] = tokens
350 - inter_data.input_positions[seq_idx] = list(range(context_len, seq_len))
480 +
481 + if isinstance(tokens, list):
482 + inter_data.input_tokens[seq_idx].extend(tokens)
483 + else:
484 + inter_data.input_tokens[seq_idx].append(tokens)
485 +
486 + if (seq_len - context_len) == 1:
487 + inter_data.input_positions[seq_idx].append(seq_len - 1)
488 + else:
489 + inter_data.input_positions[seq_idx].extend(
490 + range(context_len, seq_len))
491 +
492 inter_data.query_lens[
493 seq_idx] = seq_len - context_len if inter_data.is_prompt else 1
494
@@ -471,7 +612,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
612
613 def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata):
614 """Add a sequence group to the builder."""
474 - seq_ids = list(seq_group_metadata.seq_data.keys())
615 + seq_ids = seq_group_metadata.seq_data.keys()
616 n_seqs = len(seq_ids)
617 is_prompt = seq_group_metadata.is_prompt
618
@@ -479,12 +620,15 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
620 assert n_seqs == 1
621 self.decode_only = False
622
482 - inter_data = self.InterDataForSeqGroup(
623 + inter_data = self.init_cached_inter_data(
624 request_id=seq_group_metadata.request_id,
625 seq_ids=seq_ids,
626 is_prompt=is_prompt,
627 block_tables=seq_group_metadata.block_tables,
487 - computed_block_nums=seq_group_metadata.computed_block_nums)
628 + computed_block_nums=seq_group_metadata.computed_block_nums,
629 + reinit=True,
630 + reinit_use_defaults=True)
631 +
632 self.inter_data_list.append(inter_data)
633
634 for seq_idx in range(n_seqs):
@@ -504,18 +648,21 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
648 create on-device tensors.
649 """
650 # Combine and flatten intermediate data.
507 - input_tokens = flatten_2d_lists([
508 - flatten_2d_lists(inter_data.input_tokens)
509 - for inter_data in self.inter_data_list
510 - ])
651 + input_tokens = []
652 + for inter_data in self.inter_data_list:
653 + for cur_input_tokens in inter_data.input_tokens:
654 + input_tokens.extend(cur_input_tokens)
655 +
656 if not input_tokens:
657 # This may happen when all prefill requests hit
658 # prefix caching and there is no decode request.
659 return self.model_input_cls()
515 - input_positions = flatten_2d_lists([
516 - flatten_2d_lists(inter_data.input_positions)
517 - for inter_data in self.inter_data_list
518 - ])
660 +
661 + input_positions = []
662 + for inter_data in self.inter_data_list:
663 + for cur_input_positions in inter_data.input_positions:
664 + input_positions.extend(cur_input_positions)
665 +
666 seq_lens = []
667 max_decode_seq_len = 0
668 for inter_data in self.inter_data_list:
@@ -523,8 +670,10 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
670 if not inter_data.is_prompt:
671 max_decode_seq_len = max(max_decode_seq_len,
672 max(inter_data.seq_lens))
526 - query_lens = flatten_2d_lists(
527 - [inter_data.query_lens for inter_data in self.inter_data_list])
673 + query_lens = []
674 + for inter_data in self.inter_data_list:
675 + query_lens.extend(inter_data.query_lens)
676 +
677 # Mapping from request IDs to sequence IDs. Used for Jamba models
678 # that manages the cache by itself.
679 request_ids_to_seq_ids = {
@@ -547,8 +696,9 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
696 batch_size = graph_batch_size
697
698 # Tokens and positions.
550 - input_tokens.extend([0] * cuda_graph_pad_size)
551 - input_positions.extend([0] * cuda_graph_pad_size)
699 + if cuda_graph_pad_size:
700 + input_tokens.extend(itertools.repeat(0, cuda_graph_pad_size))
701 + input_positions.extend(itertools.repeat(0, cuda_graph_pad_size))
702 assert self.runner.device is not None
703 input_tokens_tensor = async_tensor_h2d(input_tokens, torch.long,
704 self.runner.device,
@@ -558,7 +708,8 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
708 self.runner.pin_memory)
709
710 # Sequence and query lengths.
561 - seq_lens.extend([1] * cuda_graph_pad_size)
711 + if cuda_graph_pad_size:
712 + seq_lens.extend(itertools.repeat(1, cuda_graph_pad_size))
713
714 # Attention metadata.
715 attn_metadata = self.attn_metadata_builder.build(
@@ -574,11 +725,14 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
725 flatten_2d_lists(inter_data.lora_index_mapping)
726 for inter_data in self.inter_data_list
727 ])
577 - lora_index_mapping.extend([0] * cuda_graph_pad_size)
728 + if cuda_graph_pad_size:
729 + lora_index_mapping.extend(
730 + itertools.repeat(0, cuda_graph_pad_size))
731 lora_prompt_mapping = flatten_2d_lists([
732 flatten_2d_lists(inter_data.lora_prompt_mapping)
733 for inter_data in self.inter_data_list
734 ])
735 +
736 lora_mapping = LoRAMapping(
737 **dict(index_mapping=lora_index_mapping,
738 prompt_mapping=lora_prompt_mapping,
@@ -595,7 +749,9 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
749 inter_data.prompt_adapter_index_mapping
750 for inter_data in self.inter_data_list
751 ])
598 - prompt_adapter_index_mapping.extend([0] * cuda_graph_pad_size)
752 + if cuda_graph_pad_size:
753 + prompt_adapter_index_mapping.extend(
754 + itertools.repeat(0, cuda_graph_pad_size))
755 prompt_adapter_prompt_mapping = flatten_2d_lists([
756 inter_data.prompt_adapter_prompt_mapping
757 for inter_data in self.inter_data_list
@@ -717,6 +873,11 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
873 set_cpu_offload_max_bytes(
874 int(self.cache_config.cpu_offload_gb * 1024**3))
875
876 + # Used to cache python objects
877 + self.inter_data_cache: Dict[int, PyObjectCache] = {}
878 + self.sampling_metadata_cache: SamplingMetadataCache = \
879 + SamplingMetadataCache()
880 +
881 def load_model(self) -> None:
882 logger.info("Starting to load model %s...", self.model_config.model)
883 with CudaMemoryProfiler() as m:
@@ -843,6 +1004,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1004 builder = self._builder_cls(weakref.proxy(self), finished_requests_ids)
1005 for seq_group_metadata in seq_group_metadata_list:
1006 builder.add_seq_group(seq_group_metadata)
1007 +
1008 + builder.reset_cached_inter_data()
1009 +
1010 return builder.build() # type: ignore
1011
1012 @torch.inference_mode()
@@ -1276,7 +1440,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1440 sampling_metadata = SamplingMetadata.prepare(
1441 seq_group_metadata_list, model_input.seq_lens,
1442 model_input.query_lens, self.device, self.pin_memory,
1279 - generators)
1443 + generators, self.sampling_metadata_cache)
1444 else:
1445 sampling_metadata = None
1446 is_prompt = (seq_group_metadata_list[0].is_prompt