A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add utils
Woosuk Kwon committed
Feb 9, 2023 at 11:26 UTC
5e644b912ec10ec0fac02ccdc0f9a4d8342d8f4c
1 file changed
+20
cacheflow/utils.py
new
+20
@@ -0,0 +1,20 @@
1
+import enum
2
+
3
+
4
+class Device(enum.Enum):
5
+ GPU = enum.auto()
6
+ CPU = enum.auto()
7
+
8
+
9
+class Counter:
10
+
11
+ def __init__(self, start: int = 0) -> None:
12
+ self.counter = start
13
+
14
+ def next(self) -> int:
15
+ id = self.counter
16
+ self.counter += 1
17
+ return id
18
+
19
+ def reset(self) -> None:
20
+ self.counter = 0