@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 56a8652f33
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Bugfix] store lock file in tmp directory (#3578)" (#3599)

Co-authored-by: youkaichao <youkaichao@126.com>

Woosuk Kwon committed Mar 24, 2024 at 20:06 UTC 56a8652f33955bfb5bf6766106db78eb6ff37d55
1 file changed +15 -5
vllm/model_executor/weight_utils.py
+15 -5
@@ -1,6 +1,7 @@
1 """Utilities for downloading and initializing model weights."""
2 import filelock
3 import glob
4 +import hashlib
5 import fnmatch
6 import json
7 import os
@@ -20,8 +21,12 @@ from vllm.model_executor.layers.quantization import (get_quantization_config,
21
22 logger = init_logger(__name__)
23
23 -_xdg_cache_home = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache'))
24 -_vllm_filelocks_path = os.path.join(_xdg_cache_home, 'vllm/locks/')
24 +# use system-level temp directory for file locks, so that multiple users
25 +# can share the same lock without error.
26 +# lock files in the temp directory will be automatically deleted when the
27 +# system reboots, so users will not complain about annoying lock files
28 +temp_dir = os.environ.get('TMPDIR') or os.environ.get(
29 + 'TEMP') or os.environ.get('TMP') or "/tmp/"
30
31
32 class Disabledtqdm(tqdm):
@@ -31,10 +36,15 @@ class Disabledtqdm(tqdm):
36
37
38 def get_lock(model_name_or_path: str, cache_dir: Optional[str] = None):
34 - lock_dir = cache_dir if cache_dir is not None else _vllm_filelocks_path
39 + lock_dir = cache_dir or temp_dir
40 os.makedirs(os.path.dirname(lock_dir), exist_ok=True)
36 - lock_file_name = model_name_or_path.replace("/", "-") + ".lock"
37 - lock = filelock.SoftFileLock(os.path.join(lock_dir, lock_file_name))
41 + model_name = model_name_or_path.replace("/", "-")
42 + hash_name = hashlib.sha256(model_name.encode()).hexdigest()
43 + # add hash to avoid conflict with old users' lock files
44 + lock_file_name = hash_name + model_name + ".lock"
45 + # mode 0o666 is required for the filelock to be shared across users
46 + lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name),
47 + mode=0o666)
48 return lock
49
50