@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5e644b912e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add utils

Woosuk Kwon committed Feb 9, 2023 at 11:26 UTC 5e644b912ec10ec0fac02ccdc0f9a4d8342d8f4c
1 file changed +20
cacheflow/utils.py new
+20
@@ -0,0 +1,20 @@
1 +import enum
2 +
3 +
4 +class Device(enum.Enum):
5 + GPU = enum.auto()
6 + CPU = enum.auto()
7 +
8 +
9 +class Counter:
10 +
11 + def __init__(self, start: int = 0) -> None:
12 + self.counter = start
13 +
14 + def next(self) -> int:
15 + id = self.counter
16 + self.counter += 1
17 + return id
18 +
19 + def reset(self) -> None:
20 + self.counter = 0