A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Use lru_cache for some environment detection utils (#3508)
Simon Mo committed
Mar 19, 2024 at 14:34 UTC
20478c4d3abcd0aa8a1d9ace9c76ea3a2e04cb5e
1 file changed
+5
-1
vllm/utils.py
+5
-1
@@ -11,7 +11,7 @@ from packaging.version import parse, Version
11
import psutil
12
import torch
13
import asyncio
14
-from functools import partial
14
+from functools import partial, lru_cache
15
from typing import (
16
Awaitable,
17
Callable,
@@ -120,6 +120,7 @@ def is_hip() -> bool:
120
return torch.version.hip is not None
121
122
123
+@lru_cache(maxsize=None)
124
def is_neuron() -> bool:
125
try:
126
import transformers_neuronx
@@ -128,6 +129,7 @@ def is_neuron() -> bool:
129
return transformers_neuronx is not None
130
131
132
+@lru_cache(maxsize=None)
133
def get_max_shared_memory_bytes(gpu: int = 0) -> int:
134
"""Returns the maximum shared memory per thread block in bytes."""
135
# NOTE: This import statement should be executed lazily since
@@ -151,6 +153,7 @@ def random_uuid() -> str:
153
return str(uuid.uuid4().hex)
154
155
156
+@lru_cache(maxsize=None)
157
def in_wsl() -> bool:
158
# Reference: https://github.com/microsoft/WSL/issues/4071
159
return "microsoft" in " ".join(uname()).lower()
@@ -225,6 +228,7 @@ def set_cuda_visible_devices(device_ids: List[int]) -> None:
228
os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, device_ids))
229
230
231
+@lru_cache(maxsize=None)
232
def get_nvcc_cuda_version() -> Optional[Version]:
233
cuda_home = os.environ.get('CUDA_HOME')
234
if not cuda_home: