@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9c71c97ae2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[mypy] Enable mypy type checking for `vllm/core` (#7229)

Jonathan Berkhahn committed Aug 27, 2024 at 16:11 UTC 9c71c97ae24ae6f5209a475149808e25554cfe99
9 files changed +31 -18
.github/workflows/mypy.yaml
-1
@@ -35,7 +35,6 @@ jobs:
35 mypy
36 mypy tests --follow-imports skip
37 mypy vllm/attention --follow-imports skip
38 - mypy vllm/core --follow-imports skip
38 mypy vllm/distributed --follow-imports skip
39 mypy vllm/engine --follow-imports skip
40 mypy vllm/executor --follow-imports skip
format.sh
-1
@@ -99,7 +99,6 @@ echo 'vLLM mypy:'
99 mypy --follow-imports skip # Note that this is less strict than CI
100 mypy tests --follow-imports skip
101 mypy vllm/attention --follow-imports skip
102 -mypy vllm/core --follow-imports skip
102 mypy vllm/distributed --follow-imports skip
103 mypy vllm/engine --follow-imports skip
104 mypy vllm/executor --follow-imports skip
pyproject.toml
+1
@@ -58,6 +58,7 @@ files = [
58 "vllm/adapter_commons",
59 "vllm/assets",
60 "vllm/entrypoints",
61 + "vllm/core",
62 "vllm/inputs",
63 "vllm/logging",
64 "vllm/multimodal",
vllm/block.py
+7 -2
@@ -1,9 +1,9 @@
1 """Token blocks."""
2 -from typing import List, Optional
2 +from typing import TYPE_CHECKING, Iterator, List, Optional
3
4 from vllm.utils import Device
5
6 -DEFAULT_LAST_ACCESSED_TIME = -1
6 +DEFAULT_LAST_ACCESSED_TIME: float = -1
7
8
9 class PhysicalTokenBlock:
@@ -59,6 +59,11 @@ class BlockTable:
59 def __getitem__(self, key):
60 return self._blocks[key]
61
62 + if TYPE_CHECKING:
63 +
64 + def __iter__(self) -> Iterator[PhysicalTokenBlock]:
65 + raise RuntimeError("Method should be automatically generated")
66 +
67 def __setitem__(self, key, value):
68 if isinstance(key, slice):
69 blocks = value
vllm/core/block/cpu_gpu_block_allocator.py
+1 -1
@@ -132,7 +132,7 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
132
133 def allocate_immutable_blocks(self, prev_block: Optional[Block],
134 block_token_ids: List[List[int]],
135 - device: Optional[Device]) -> List[Block]:
135 + device: Device) -> List[Block]:
136 """Allocates a new group of immutable blocks with the provided block
137 token IDs on the specified device.
138
vllm/core/block_manager_v1.py
+4 -3
@@ -278,7 +278,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
278 # request ID
279 self.cross_block_tables: Dict[str, BlockTable] = {}
280
281 - def _get_seq_num_required_blocks(self, seq: Sequence) -> int:
281 + def _get_seq_num_required_blocks(self, seq: Optional[Sequence]) -> int:
282 return 0 if seq is None else seq.n_blocks
283
284 def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
@@ -310,13 +310,14 @@ class BlockSpaceManagerV1(BlockSpaceManager):
310 return AllocStatus.LATER
311
312 def _allocate_sequence(self, \
313 - seq: Sequence, \
313 + seq: Optional[Sequence], \
314 ref_count: int, \
315 is_encoder_decoder: bool = True) -> BlockTable:
316 # Allocate new physical token blocks that will store the prompt tokens.
317 - num_prompt_blocks = seq.n_blocks
317 + num_prompt_blocks = self._get_seq_num_required_blocks(seq)
318
319 block_table: BlockTable = BlockTable()
320 + assert seq is not None
321 for logical_idx in range(num_prompt_blocks):
322 if (self.block_sliding_window is not None
323 and logical_idx >= self.block_sliding_window):
vllm/core/block_manager_v2.py
+6 -2
@@ -120,8 +120,10 @@ class BlockSpaceManagerV2(BlockSpaceManager):
120 )
121
122 if seq_group.is_encoder_decoder():
123 + encoder_seq = seq_group.get_encoder_seq()
124 + assert encoder_seq is not None
125 num_required_blocks += BlockTable.get_num_required_blocks(
124 - seq_group.get_encoder_seq().get_token_ids(),
126 + encoder_seq.get_token_ids(),
127 block_size=self.block_size,
128 )
129
@@ -189,7 +191,9 @@ class BlockSpaceManagerV2(BlockSpaceManager):
191 check_no_caching_or_swa_for_blockmgr_encdec(self, seq_group)
192
193 if seq_group.is_encoder_decoder():
192 - block_table = self._allocate_sequence(seq_group.get_encoder_seq())
194 + encoder_seq = seq_group.get_encoder_seq()
195 + assert encoder_seq is not None
196 + block_table = self._allocate_sequence(encoder_seq)
197 self.cross_block_tables[request_id] = block_table
198
199 def can_append_slots(self, seq_group: SequenceGroup,
vllm/core/embedding_model_block_manager.py
+2 -2
@@ -77,8 +77,8 @@ class EmbeddingModelBlockSpaceManager(BlockSpaceManager):
77 pass
78
79 def get_common_computed_block_ids(self,
80 - seq_group: SequenceGroup) -> List[int]:
81 - return None # type: ignore
80 + seq_group: List[Sequence]) -> List[int]:
81 + return []
82
83 def mark_blocks_as_computed(self, seq_group: SequenceGroup):
84 pass
vllm/core/scheduler.py
+10 -6
@@ -221,10 +221,10 @@ class SchedulerSwappedInOutputs:
221 """
222 # Selected sequences that are going to be swapped in and is in a
223 # decoding phase.
224 - decode_seq_groups: List[SequenceGroup]
224 + decode_seq_groups: List[ScheduledSequenceGroup]
225 # Selected sequences that are going to be swapped in and in a prefill
226 # phase. I.e., it means the prefill has been chunked.
227 - prefill_seq_groups: List[SequenceGroup]
227 + prefill_seq_groups: List[ScheduledSequenceGroup]
228 # The blocks to swap in.
229 blocks_to_swap_in: List[Tuple[int, int]]
230 # The blocks to copy.
@@ -254,7 +254,7 @@ class SchedulerPrefillOutputs:
254 to be recomputed from scratch.
255 """
256 # Selected sequences for prefill.
257 - seq_groups: List[SequenceGroup]
257 + seq_groups: List[ScheduledSequenceGroup]
258 # Ignored sequence groups.
259 ignored_seq_groups: List[SequenceGroup]
260 num_lookahead_slots: int
@@ -289,7 +289,9 @@ def scheduler_running_outputs_builder():
289
290
291 def scheduled_seq_group_builder():
292 - return ScheduledSequenceGroup(seq_group=None, token_chunk_size=0)
292 + return ScheduledSequenceGroup(SequenceGroup("", [], -1),
293 + token_chunk_size=0)
294 + # return ScheduledSequenceGroup(seq_group=None, token_chunk_size=0)
295
296
297 class Scheduler:
@@ -791,7 +793,7 @@ class Scheduler:
793 SchedulerPrefillOutputs.
794 """
795 ignored_seq_groups: List[SequenceGroup] = []
794 - seq_groups: List[SequenceGroup] = []
796 + seq_groups: List[ScheduledSequenceGroup] = []
797
798 waiting_queue = self.waiting
799
@@ -1130,7 +1132,9 @@ class Scheduler:
1132
1133 if seq_group.is_encoder_decoder():
1134 # Encoder associated with SequenceGroup
1133 - encoder_seq_data = seq_group.get_encoder_seq().data
1135 + encoder_seq = seq_group.get_encoder_seq()
1136 + assert encoder_seq is not None
1137 + encoder_seq_data = encoder_seq.data
1138 # Block table for cross-attention
1139 # Also managed at SequenceGroup level
1140 cross_block_table = self.block_manager.get_cross_block_table(