A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Bugfix] store lock file in tmp directory (#3578)" (#3599)
Co-authored-by: youkaichao <youkaichao@126.com>
Woosuk Kwon committed
Mar 24, 2024 at 20:06 UTC
56a8652f33955bfb5bf6766106db78eb6ff37d55
1 file changed
+15
-5
vllm/model_executor/weight_utils.py
+15
-5
@@ -1,6 +1,7 @@
1
"""Utilities for downloading and initializing model weights."""
2
import filelock
3
import glob
4
+import hashlib
5
import fnmatch
6
import json
7
import os
@@ -20,8 +21,12 @@ from vllm.model_executor.layers.quantization import (get_quantization_config,
21
22
logger = init_logger(__name__)
23
23
-_xdg_cache_home = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache'))
24
-_vllm_filelocks_path = os.path.join(_xdg_cache_home, 'vllm/locks/')
24
+# use system-level temp directory for file locks, so that multiple users
25
+# can share the same lock without error.
26
+# lock files in the temp directory will be automatically deleted when the
27
+# system reboots, so users will not complain about annoying lock files
28
+temp_dir = os.environ.get('TMPDIR') or os.environ.get(
29
+ 'TEMP') or os.environ.get('TMP') or "/tmp/"
30
31
32
class Disabledtqdm(tqdm):
@@ -31,10 +36,15 @@ class Disabledtqdm(tqdm):
36
37
38
def get_lock(model_name_or_path: str, cache_dir: Optional[str] = None):
34
- lock_dir = cache_dir if cache_dir is not None else _vllm_filelocks_path
39
+ lock_dir = cache_dir or temp_dir
40
os.makedirs(os.path.dirname(lock_dir), exist_ok=True)
36
- lock_file_name = model_name_or_path.replace("/", "-") + ".lock"
37
- lock = filelock.SoftFileLock(os.path.join(lock_dir, lock_file_name))
41
+ model_name = model_name_or_path.replace("/", "-")
42
+ hash_name = hashlib.sha256(model_name.encode()).hexdigest()
43
+ # add hash to avoid conflict with old users' lock files
44
+ lock_file_name = hash_name + model_name + ".lock"
45
+ # mode 0o666 is required for the filelock to be shared across users
46
+ lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name),
47
+ mode=0o666)
48
return lock
49
50