@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3363c27d19
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add __repr__

Woosuk Kwon committed Feb 14, 2023 at 09:34 UTC 3363c27d1954fd63741020dbf5e48bc9b9735cd7
2 files changed +14 -3
cacheflow/block.py
+5
@@ -49,3 +49,8 @@ class PhysicalTokenBlock:
49 self.block_size = block_size
50
51 self.ref_count = 0
52 +
53 + def __repr__(self) -> str:
54 + return (f'PhysicalTokenBlock(device={self.device}, '
55 + f'block_number={self.block_number}, '
56 + f'ref_count={self.ref_count})')
cacheflow/sequence.py
+9 -3
@@ -2,7 +2,6 @@ import enum
2 from typing import List, Optional
3
4 from cacheflow.block import LogicalTokenBlock
5 -from cacheflow.decoding import DecodingParams
5
6
7 class SequenceStatus(enum.Enum):
@@ -58,6 +57,11 @@ class Sequence:
57 token_ids.extend(block.get_token_ids())
58 return token_ids
59
60 + def __repr__(self) -> str:
61 + return (f'Sequence(seq_id={self.seq_id}, '
62 + f'status={self.status.name}, '
63 + f'num_blocks={len(self.logical_token_blocks)})')
64 +
65
66 class SequenceGroup:
67
@@ -65,11 +69,9 @@ class SequenceGroup:
69 self,
70 group_id: int,
71 seqs: List[Sequence],
68 - decoding_params: DecodingParams,
72 ) -> None:
73 self.group_id = group_id
74 self.seqs = seqs
72 - self.decoding_params = decoding_params
75
76 def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
77 if status is None:
@@ -82,3 +84,7 @@ class SequenceGroup:
84 if seq.seq_id == seq_id:
85 return seq
86 raise ValueError(f'Sequence {seq_id} not found.')
87 +
88 + def __repr__(self) -> str:
89 + return (f'SequenceGroup(group_id={self.group_id}, '
90 + f'num_seqs={len(self.seqs)})')