A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[mypy] Enable mypy type checking for `vllm/core` (#7229)
Jonathan Berkhahn committed
Aug 27, 2024 at 16:11 UTC
9c71c97ae24ae6f5209a475149808e25554cfe99
9 files changed
+31
-18
.github/workflows/mypy.yaml
-1
@@ -35,7 +35,6 @@ jobs:
35
mypy
36
mypy tests --follow-imports skip
37
mypy vllm/attention --follow-imports skip
38
- mypy vllm/core --follow-imports skip
38
mypy vllm/distributed --follow-imports skip
39
mypy vllm/engine --follow-imports skip
40
mypy vllm/executor --follow-imports skip
format.sh
-1
@@ -99,7 +99,6 @@ echo 'vLLM mypy:'
99
mypy --follow-imports skip # Note that this is less strict than CI
100
mypy tests --follow-imports skip
101
mypy vllm/attention --follow-imports skip
102
-mypy vllm/core --follow-imports skip
102
mypy vllm/distributed --follow-imports skip
103
mypy vllm/engine --follow-imports skip
104
mypy vllm/executor --follow-imports skip
pyproject.toml
+1
@@ -58,6 +58,7 @@ files = [
58
"vllm/adapter_commons",
59
"vllm/assets",
60
"vllm/entrypoints",
61
+ "vllm/core",
62
"vllm/inputs",
63
"vllm/logging",
64
"vllm/multimodal",
vllm/block.py
+7
-2
@@ -1,9 +1,9 @@
1
"""Token blocks."""
2
-from typing import List, Optional
2
+from typing import TYPE_CHECKING, Iterator, List, Optional
3
4
from vllm.utils import Device
5
6
-DEFAULT_LAST_ACCESSED_TIME = -1
6
+DEFAULT_LAST_ACCESSED_TIME: float = -1
7
8
9
class PhysicalTokenBlock:
@@ -59,6 +59,11 @@ class BlockTable:
59
def __getitem__(self, key):
60
return self._blocks[key]
61
62
+ if TYPE_CHECKING:
63
+
64
+ def __iter__(self) -> Iterator[PhysicalTokenBlock]:
65
+ raise RuntimeError("Method should be automatically generated")
66
+
67
def __setitem__(self, key, value):
68
if isinstance(key, slice):
69
blocks = value
vllm/core/block/cpu_gpu_block_allocator.py
+1
-1
@@ -132,7 +132,7 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
132
133
def allocate_immutable_blocks(self, prev_block: Optional[Block],
134
block_token_ids: List[List[int]],
135
- device: Optional[Device]) -> List[Block]:
135
+ device: Device) -> List[Block]:
136
"""Allocates a new group of immutable blocks with the provided block
137
token IDs on the specified device.
138
vllm/core/block_manager_v1.py
+4
-3
@@ -278,7 +278,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
278
# request ID
279
self.cross_block_tables: Dict[str, BlockTable] = {}
280
281
- def _get_seq_num_required_blocks(self, seq: Sequence) -> int:
281
+ def _get_seq_num_required_blocks(self, seq: Optional[Sequence]) -> int:
282
return 0 if seq is None else seq.n_blocks
283
284
def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
@@ -310,13 +310,14 @@ class BlockSpaceManagerV1(BlockSpaceManager):
310
return AllocStatus.LATER
311
312
def _allocate_sequence(self, \
313
- seq: Sequence, \
313
+ seq: Optional[Sequence], \
314
ref_count: int, \
315
is_encoder_decoder: bool = True) -> BlockTable:
316
# Allocate new physical token blocks that will store the prompt tokens.
317
- num_prompt_blocks = seq.n_blocks
317
+ num_prompt_blocks = self._get_seq_num_required_blocks(seq)
318
319
block_table: BlockTable = BlockTable()
320
+ assert seq is not None
321
for logical_idx in range(num_prompt_blocks):
322
if (self.block_sliding_window is not None
323
and logical_idx >= self.block_sliding_window):
vllm/core/block_manager_v2.py
+6
-2
@@ -120,8 +120,10 @@ class BlockSpaceManagerV2(BlockSpaceManager):
120
)
121
122
if seq_group.is_encoder_decoder():
123
+ encoder_seq = seq_group.get_encoder_seq()
124
+ assert encoder_seq is not None
125
num_required_blocks += BlockTable.get_num_required_blocks(
124
- seq_group.get_encoder_seq().get_token_ids(),
126
+ encoder_seq.get_token_ids(),
127
block_size=self.block_size,
128
)
129
@@ -189,7 +191,9 @@ class BlockSpaceManagerV2(BlockSpaceManager):
191
check_no_caching_or_swa_for_blockmgr_encdec(self, seq_group)
192
193
if seq_group.is_encoder_decoder():
192
- block_table = self._allocate_sequence(seq_group.get_encoder_seq())
194
+ encoder_seq = seq_group.get_encoder_seq()
195
+ assert encoder_seq is not None
196
+ block_table = self._allocate_sequence(encoder_seq)
197
self.cross_block_tables[request_id] = block_table
198
199
def can_append_slots(self, seq_group: SequenceGroup,
vllm/core/embedding_model_block_manager.py
+2
-2
@@ -77,8 +77,8 @@ class EmbeddingModelBlockSpaceManager(BlockSpaceManager):
77
pass
78
79
def get_common_computed_block_ids(self,
80
- seq_group: SequenceGroup) -> List[int]:
81
- return None # type: ignore
80
+ seq_group: List[Sequence]) -> List[int]:
81
+ return []
82
83
def mark_blocks_as_computed(self, seq_group: SequenceGroup):
84
pass
vllm/core/scheduler.py
+10
-6
@@ -221,10 +221,10 @@ class SchedulerSwappedInOutputs:
221
"""
222
# Selected sequences that are going to be swapped in and is in a
223
# decoding phase.
224
- decode_seq_groups: List[SequenceGroup]
224
+ decode_seq_groups: List[ScheduledSequenceGroup]
225
# Selected sequences that are going to be swapped in and in a prefill
226
# phase. I.e., it means the prefill has been chunked.
227
- prefill_seq_groups: List[SequenceGroup]
227
+ prefill_seq_groups: List[ScheduledSequenceGroup]
228
# The blocks to swap in.
229
blocks_to_swap_in: List[Tuple[int, int]]
230
# The blocks to copy.
@@ -254,7 +254,7 @@ class SchedulerPrefillOutputs:
254
to be recomputed from scratch.
255
"""
256
# Selected sequences for prefill.
257
- seq_groups: List[SequenceGroup]
257
+ seq_groups: List[ScheduledSequenceGroup]
258
# Ignored sequence groups.
259
ignored_seq_groups: List[SequenceGroup]
260
num_lookahead_slots: int
@@ -289,7 +289,9 @@ def scheduler_running_outputs_builder():
289
290
291
def scheduled_seq_group_builder():
292
- return ScheduledSequenceGroup(seq_group=None, token_chunk_size=0)
292
+ return ScheduledSequenceGroup(SequenceGroup("", [], -1),
293
+ token_chunk_size=0)
294
+ # return ScheduledSequenceGroup(seq_group=None, token_chunk_size=0)
295
296
297
class Scheduler:
@@ -791,7 +793,7 @@ class Scheduler:
793
SchedulerPrefillOutputs.
794
"""
795
ignored_seq_groups: List[SequenceGroup] = []
794
- seq_groups: List[SequenceGroup] = []
796
+ seq_groups: List[ScheduledSequenceGroup] = []
797
798
waiting_queue = self.waiting
799
@@ -1130,7 +1132,9 @@ class Scheduler:
1132
1133
if seq_group.is_encoder_decoder():
1134
# Encoder associated with SequenceGroup
1133
- encoder_seq_data = seq_group.get_encoder_seq().data
1135
+ encoder_seq = seq_group.get_encoder_seq()
1136
+ assert encoder_seq is not None
1137
+ encoder_seq_data = encoder_seq.data
1138
# Block table for cross-attention
1139
# Also managed at SequenceGroup level
1140
cross_block_table = self.block_manager.get_cross_block_table(