A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Move model filelocks from `/tmp/` to `~/.cache/vllm/locks/` dir (#3241)
Michael Goin committed
Mar 8, 2024 at 13:33 UTC
c2c5e0909ad4457ad542117939c2629ebe2db609
1 file changed
+5
-1
vllm/model_executor/weight_utils.py
+5
-1
@@ -20,6 +20,9 @@ from vllm.model_executor.layers.quantization import (get_quantization_config,
20
21
logger = init_logger(__name__)
22
23
+_xdg_cache_home = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache'))
24
+_vllm_filelocks_path = os.path.join(_xdg_cache_home, 'vllm/locks/')
25
+
26
27
class Disabledtqdm(tqdm):
28
@@ -28,7 +31,8 @@ class Disabledtqdm(tqdm):
31
32
33
def get_lock(model_name_or_path: str, cache_dir: Optional[str] = None):
31
- lock_dir = cache_dir if cache_dir is not None else "/tmp"
34
+ lock_dir = cache_dir if cache_dir is not None else _vllm_filelocks_path
35
+ os.makedirs(os.path.dirname(lock_dir), exist_ok=True)
36
lock_file_name = model_name_or_path.replace("/", "-") + ".lock"
37
lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name))
38
return lock