A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Misc] Separate out `utils.counter` and move `utils.Device` to engine (#27588)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Oct 28, 2025 at 20:20 UTC
f58d9b64044e465b85e9280882c738e11b59b2d6
8 files changed
+59
-54
vllm/engine/protocol.py
+6
-1
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import enum
5
from abc import ABC, abstractmethod
6
from collections.abc import AsyncGenerator, Iterable, Mapping
7
from typing import Any
@@ -15,13 +16,17 @@ from vllm.pooling_params import PoolingParams
16
from vllm.sampling_params import SamplingParams
17
from vllm.tasks import SupportedTask
18
from vllm.transformers_utils.tokenizer import AnyTokenizer
18
-from vllm.utils import Device
19
from vllm.v1.engine import EngineCoreRequest
20
from vllm.v1.engine.processor import Processor
21
22
logger = init_logger(__name__)
23
24
25
+class Device(enum.Enum):
26
+ GPU = enum.auto()
27
+ CPU = enum.auto()
28
+
29
+
30
class EngineClient(ABC):
31
"""Protocol class for Clients to Engine"""
32
vllm/entrypoints/llm.py
+4
-3
@@ -31,6 +31,7 @@ from vllm.config.model import (
31
TokenizerMode,
32
)
33
from vllm.engine.arg_utils import EngineArgs
34
+from vllm.engine.protocol import Device
35
from vllm.entrypoints.chat_utils import (
36
ChatCompletionMessageParam,
37
ChatTemplateContentFormatOption,
@@ -75,8 +76,8 @@ from vllm.transformers_utils.tokenizer import (
76
get_cached_tokenizer,
77
)
78
from vllm.usage.usage_lib import UsageContext
78
-from vllm.utils import Counter, Device
79
from vllm.utils.collection_utils import as_iter, is_list_of
80
+from vllm.utils.counter import Counter
81
from vllm.v1.engine import EngineCoreRequest
82
from vllm.v1.engine.llm_engine import LLMEngine
83
from vllm.v1.sample.logits_processor import LogitsProcessor
@@ -1490,8 +1491,8 @@ class LLM:
1491
def stop_profile(self) -> None:
1492
self.llm_engine.stop_profile()
1493
1493
- def reset_prefix_cache(self, device: Device | None = None) -> bool:
1494
- return self.llm_engine.reset_prefix_cache(device)
1494
+ def reset_prefix_cache(self, device: Device | None = None) -> None:
1495
+ self.llm_engine.reset_prefix_cache(device)
1496
1497
def sleep(self, level: int = 1):
1498
"""
vllm/entrypoints/openai/api_server.py
+1
-2
@@ -40,7 +40,7 @@ from typing_extensions import assert_never
40
import vllm.envs as envs
41
from vllm.config import VllmConfig
42
from vllm.engine.arg_utils import AsyncEngineArgs
43
-from vllm.engine.protocol import EngineClient
43
+from vllm.engine.protocol import Device, EngineClient
44
from vllm.entrypoints.launcher import serve_http
45
from vllm.entrypoints.logger import RequestLogger
46
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
@@ -108,7 +108,6 @@ from vllm.entrypoints.utils import (
108
from vllm.logger import init_logger
109
from vllm.reasoning import ReasoningParserManager
110
from vllm.usage.usage_lib import UsageContext
111
-from vllm.utils import Device
111
from vllm.utils.argparse_utils import FlexibleArgumentParser
112
from vllm.utils.network_utils import is_valid_ipv6_address
113
from vllm.utils.system_utils import decorate_logs, set_ulimit
vllm/entrypoints/openai/serving_models.py
+1
-1
@@ -19,7 +19,7 @@ from vllm.entrypoints.openai.protocol import (
19
from vllm.logger import init_logger
20
from vllm.lora.request import LoRARequest
21
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
22
-from vllm.utils import AtomicCounter
22
+from vllm.utils.counter import AtomicCounter
23
24
logger = init_logger(__name__)
25
vllm/utils/__init__.py
-44
@@ -3,7 +3,6 @@
3
4
import enum
5
import inspect
6
-import threading
6
import uuid
7
import warnings
8
from functools import wraps
@@ -68,54 +67,11 @@ STR_INVALID_VAL: str = "INVALID"
67
T = TypeVar("T")
68
69
71
-class Device(enum.Enum):
72
- GPU = enum.auto()
73
- CPU = enum.auto()
74
-
75
-
70
class LayerBlockType(enum.Enum):
71
attention = "attention"
72
mamba = "mamba"
73
74
81
-class Counter:
82
- def __init__(self, start: int = 0) -> None:
83
- self.counter = start
84
-
85
- def __next__(self) -> int:
86
- i = self.counter
87
- self.counter += 1
88
- return i
89
-
90
- def reset(self) -> None:
91
- self.counter = 0
92
-
93
-
94
-class AtomicCounter:
95
- """An atomic, thread-safe counter"""
96
-
97
- def __init__(self, initial=0):
98
- """Initialize a new atomic counter to given initial value"""
99
- self._value = initial
100
- self._lock = threading.Lock()
101
-
102
- def inc(self, num=1):
103
- """Atomically increment the counter by num and return the new value"""
104
- with self._lock:
105
- self._value += num
106
- return self._value
107
-
108
- def dec(self, num=1):
109
- """Atomically decrement the counter by num and return the new value"""
110
- with self._lock:
111
- self._value -= num
112
- return self._value
113
-
114
- @property
115
- def value(self):
116
- return self._value
117
-
118
-
75
def random_uuid() -> str:
76
return str(uuid.uuid4().hex)
77
vllm/utils/counter.py
new
+45
@@ -0,0 +1,45 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import threading
4
+
5
+
6
+class Counter:
7
+ def __init__(self, start: int = 0) -> None:
8
+ super().__init__()
9
+
10
+ self.counter = start
11
+
12
+ def __next__(self) -> int:
13
+ i = self.counter
14
+ self.counter += 1
15
+ return i
16
+
17
+ def reset(self) -> None:
18
+ self.counter = 0
19
+
20
+
21
+class AtomicCounter:
22
+ """An atomic, thread-safe counter"""
23
+
24
+ def __init__(self, initial: int = 0) -> None:
25
+ """Initialize a new atomic counter to given initial value"""
26
+ super().__init__()
27
+
28
+ self._value = initial
29
+ self._lock = threading.Lock()
30
+
31
+ @property
32
+ def value(self) -> int:
33
+ return self._value
34
+
35
+ def inc(self, num: int = 1) -> int:
36
+ """Atomically increment the counter by num and return the new value"""
37
+ with self._lock:
38
+ self._value += num
39
+ return self._value
40
+
41
+ def dec(self, num: int = 1) -> int:
42
+ """Atomically decrement the counter by num and return the new value"""
43
+ with self._lock:
44
+ self._value -= num
45
+ return self._value
vllm/v1/engine/async_llm.py
+1
-2
@@ -14,7 +14,7 @@ import torch
14
import vllm.envs as envs
15
from vllm.config import VllmConfig
16
from vllm.engine.arg_utils import AsyncEngineArgs
17
-from vllm.engine.protocol import EngineClient
17
+from vllm.engine.protocol import Device, EngineClient
18
from vllm.entrypoints.utils import _validate_truncation_size
19
from vllm.inputs import PromptType
20
from vllm.logger import init_logger
@@ -29,7 +29,6 @@ from vllm.tracing import init_tracer
29
from vllm.transformers_utils.config import maybe_register_config_serialize_by_value
30
from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs
31
from vllm.usage.usage_lib import UsageContext
32
-from vllm.utils import Device
32
from vllm.utils.async_utils import cancel_task_threadsafe
33
from vllm.utils.collection_utils import as_list
34
from vllm.utils.func_utils import deprecate_kwargs
vllm/v1/engine/llm_engine.py
+1
-1
@@ -14,6 +14,7 @@ from vllm.config import ParallelConfig, VllmConfig
14
from vllm.distributed import stateless_destroy_torch_distributed_process_group
15
from vllm.distributed.parallel_state import get_dp_group
16
from vllm.engine.arg_utils import EngineArgs
17
+from vllm.engine.protocol import Device
18
from vllm.inputs import PromptType
19
from vllm.logger import init_logger
20
from vllm.lora.request import LoRARequest
@@ -26,7 +27,6 @@ from vllm.tasks import SupportedTask
27
from vllm.tracing import init_tracer
28
from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs
29
from vllm.usage.usage_lib import UsageContext
29
-from vllm.utils import Device
30
from vllm.v1.engine import EngineCoreRequest
31
from vllm.v1.engine.core_client import EngineCoreClient
32
from vllm.v1.engine.output_processor import OutputProcessor