A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add __repr__
Woosuk Kwon committed
Feb 14, 2023 at 09:34 UTC
3363c27d1954fd63741020dbf5e48bc9b9735cd7
2 files changed
+14
-3
cacheflow/block.py
+5
@@ -49,3 +49,8 @@ class PhysicalTokenBlock:
49
self.block_size = block_size
50
51
self.ref_count = 0
52
+
53
+ def __repr__(self) -> str:
54
+ return (f'PhysicalTokenBlock(device={self.device}, '
55
+ f'block_number={self.block_number}, '
56
+ f'ref_count={self.ref_count})')
cacheflow/sequence.py
+9
-3
@@ -2,7 +2,6 @@ import enum
2
from typing import List, Optional
3
4
from cacheflow.block import LogicalTokenBlock
5
-from cacheflow.decoding import DecodingParams
5
6
7
class SequenceStatus(enum.Enum):
@@ -58,6 +57,11 @@ class Sequence:
57
token_ids.extend(block.get_token_ids())
58
return token_ids
59
60
+ def __repr__(self) -> str:
61
+ return (f'Sequence(seq_id={self.seq_id}, '
62
+ f'status={self.status.name}, '
63
+ f'num_blocks={len(self.logical_token_blocks)})')
64
+
65
66
class SequenceGroup:
67
@@ -65,11 +69,9 @@ class SequenceGroup:
69
self,
70
group_id: int,
71
seqs: List[Sequence],
68
- decoding_params: DecodingParams,
72
) -> None:
73
self.group_id = group_id
74
self.seqs = seqs
72
- self.decoding_params = decoding_params
75
76
def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
77
if status is None:
@@ -82,3 +84,7 @@ class SequenceGroup:
84
if seq.seq_id == seq_id:
85
return seq
86
raise ValueError(f'Sequence {seq_id} not found.')
87
+
88
+ def __repr__(self) -> str:
89
+ return (f'SequenceGroup(group_id={self.group_id}, '
90
+ f'num_seqs={len(self.seqs)})')