@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b37cdce2b1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Core] Cache some utils (#3474)

Antoni Baum committed Mar 18, 2024 at 17:14 UTC b37cdce2b1125ac06829c2606be1e26d75b5a505
1 file changed +5
vllm/utils.py
+5
@@ -4,6 +4,7 @@ import socket
4 import subprocess
5 import uuid
6 import gc
7 +from functools import cache
8 from platform import uname
9 from typing import List, Tuple, Union
10 from packaging.version import parse, Version
@@ -120,6 +121,7 @@ def is_hip() -> bool:
121 return torch.version.hip is not None
122
123
124 +@cache
125 def is_neuron() -> bool:
126 try:
127 import transformers_neuronx
@@ -128,6 +130,7 @@ def is_neuron() -> bool:
130 return transformers_neuronx is not None
131
132
133 +@cache
134 def get_max_shared_memory_bytes(gpu: int = 0) -> int:
135 """Returns the maximum shared memory per thread block in bytes."""
136 # NOTE: This import statement should be executed lazily since
@@ -151,6 +154,7 @@ def random_uuid() -> str:
154 return str(uuid.uuid4().hex)
155
156
157 +@cache
158 def in_wsl() -> bool:
159 # Reference: https://github.com/microsoft/WSL/issues/4071
160 return "microsoft" in " ".join(uname()).lower()
@@ -225,6 +229,7 @@ def set_cuda_visible_devices(device_ids: List[int]) -> None:
229 os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, device_ids))
230
231
232 +@cache
233 def get_nvcc_cuda_version() -> Optional[Version]:
234 cuda_home = os.environ.get('CUDA_HOME')
235 if not cuda_home: