@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c2c5e0909a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Move model filelocks from `/tmp/` to `~/.cache/vllm/locks/` dir (#3241)

Michael Goin committed Mar 8, 2024 at 13:33 UTC c2c5e0909ad4457ad542117939c2629ebe2db609
1 file changed +5 -1
vllm/model_executor/weight_utils.py
+5 -1
@@ -20,6 +20,9 @@ from vllm.model_executor.layers.quantization import (get_quantization_config,
20
21 logger = init_logger(__name__)
22
23 +_xdg_cache_home = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache'))
24 +_vllm_filelocks_path = os.path.join(_xdg_cache_home, 'vllm/locks/')
25 +
26
27 class Disabledtqdm(tqdm):
28
@@ -28,7 +31,8 @@ class Disabledtqdm(tqdm):
31
32
33 def get_lock(model_name_or_path: str, cache_dir: Optional[str] = None):
31 - lock_dir = cache_dir if cache_dir is not None else "/tmp"
34 + lock_dir = cache_dir if cache_dir is not None else _vllm_filelocks_path
35 + os.makedirs(os.path.dirname(lock_dir), exist_ok=True)
36 lock_file_name = model_name_or_path.replace("/", "-") + ".lock"
37 lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name))
38 return lock