A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] add cache model when from object storage get model (#24764)
Signed-off-by: rongfu.leng <rongfu.leng@daocloud.io>
rongfu.leng committed
Sep 25, 2025 at 02:11 UTC
2dda3e35d054235b0c2170df359b42ec25b4fe2c
2 files changed
+14
-8
vllm/envs.py
+6
@@ -64,6 +64,7 @@ if TYPE_CHECKING:
64
VLLM_XLA_USE_SPMD: bool = False
65
VLLM_WORKER_MULTIPROC_METHOD: Literal["fork", "spawn"] = "fork"
66
VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets")
67
+ VLLM_ASSETS_CACHE_MODEL_CLEAN: bool = False
68
VLLM_IMAGE_FETCH_TIMEOUT: int = 5
69
VLLM_VIDEO_FETCH_TIMEOUT: int = 30
70
VLLM_AUDIO_FETCH_TIMEOUT: int = 10
@@ -699,6 +700,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
700
os.path.join(get_default_cache_root(), "vllm", "assets"),
701
)),
702
703
+ # If the env var is set, we will clean model file in
704
+ # this path $VLLM_ASSETS_CACHE/model_streamer/$model_name
705
+ "VLLM_ASSETS_CACHE_MODEL_CLEAN":
706
+ lambda: bool(int(os.getenv("VLLM_ASSETS_CACHE_MODEL_CLEAN", "0"))),
707
+
708
# Timeout for fetching images when serving multimodal models
709
# Default is 5 seconds
710
"VLLM_IMAGE_FETCH_TIMEOUT":
vllm/transformers_utils/runai_utils.py
+8
-8
@@ -5,9 +5,10 @@ import hashlib
5
import os
6
import shutil
7
import signal
8
-import tempfile
8
from typing import Optional
9
10
+from vllm import envs
11
+from vllm.assets.base import get_cache_dir
12
from vllm.logger import init_logger
13
from vllm.utils import PlaceholderModule
14
@@ -58,20 +59,19 @@ class ObjectStorageModel:
59
"""
60
61
def __init__(self, url: str) -> None:
61
- for sig in (signal.SIGINT, signal.SIGTERM):
62
- existing_handler = signal.getsignal(sig)
63
- signal.signal(sig, self._close_by_signal(existing_handler))
62
+ if envs.VLLM_ASSETS_CACHE_MODEL_CLEAN:
63
+ for sig in (signal.SIGINT, signal.SIGTERM):
64
+ existing_handler = signal.getsignal(sig)
65
+ signal.signal(sig, self._close_by_signal(existing_handler))
66
67
dir_name = os.path.join(
66
- tempfile.gettempdir(),
68
+ get_cache_dir(), "model_streamer",
69
hashlib.sha256(str(url).encode()).hexdigest()[:8])
70
if os.path.exists(dir_name):
71
shutil.rmtree(dir_name)
72
os.makedirs(dir_name)
73
self.dir = dir_name
72
-
73
- def __del__(self):
74
- self._close()
74
+ logger.debug("Init object storage, model cache path is: %s", dir_name)
75
76
def _close(self) -> None:
77
if os.path.exists(self.dir):