@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 20478c4d3a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Use lru_cache for some environment detection utils (#3508)

Simon Mo committed Mar 19, 2024 at 14:34 UTC 20478c4d3abcd0aa8a1d9ace9c76ea3a2e04cb5e
1 file changed +5 -1
vllm/utils.py
+5 -1
@@ -11,7 +11,7 @@ from packaging.version import parse, Version
11 import psutil
12 import torch
13 import asyncio
14 -from functools import partial
14 +from functools import partial, lru_cache
15 from typing import (
16 Awaitable,
17 Callable,
@@ -120,6 +120,7 @@ def is_hip() -> bool:
120 return torch.version.hip is not None
121
122
123 +@lru_cache(maxsize=None)
124 def is_neuron() -> bool:
125 try:
126 import transformers_neuronx
@@ -128,6 +129,7 @@ def is_neuron() -> bool:
129 return transformers_neuronx is not None
130
131
132 +@lru_cache(maxsize=None)
133 def get_max_shared_memory_bytes(gpu: int = 0) -> int:
134 """Returns the maximum shared memory per thread block in bytes."""
135 # NOTE: This import statement should be executed lazily since
@@ -151,6 +153,7 @@ def random_uuid() -> str:
153 return str(uuid.uuid4().hex)
154
155
156 +@lru_cache(maxsize=None)
157 def in_wsl() -> bool:
158 # Reference: https://github.com/microsoft/WSL/issues/4071
159 return "microsoft" in " ".join(uname()).lower()
@@ -225,6 +228,7 @@ def set_cuda_visible_devices(device_ids: List[int]) -> None:
228 os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, device_ids))
229
230
231 +@lru_cache(maxsize=None)
232 def get_nvcc_cuda_version() -> Optional[Version]:
233 cuda_home = os.environ.get('CUDA_HOME')
234 if not cuda_home: