A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[V1] Consolidate MM cache size to vllm.envs (#13239)
Roger Wang committed
Feb 13, 2025 at 20:19 UTC
dd5ede444032cfb00d8377722b7f24ec9157666b
3 files changed
+18
-11
vllm/envs.py
+9
-2
@@ -55,6 +55,7 @@ if TYPE_CHECKING:
55
VLLM_IMAGE_FETCH_TIMEOUT: int = 5
56
VLLM_VIDEO_FETCH_TIMEOUT: int = 30
57
VLLM_AUDIO_FETCH_TIMEOUT: int = 10
58
+ VLLM_MM_INPUT_CACHE_SIZE: int = 256
59
VLLM_TARGET_DEVICE: str = "cuda"
60
MAX_JOBS: Optional[str] = None
61
NVCC_THREADS: Optional[str] = None
@@ -401,15 +402,21 @@ environment_variables: Dict[str, Callable[[], Any]] = {
402
lambda: int(os.getenv("VLLM_IMAGE_FETCH_TIMEOUT", "5")),
403
404
# Timeout for fetching videos when serving multimodal models
404
- # Default is 15 seconds
405
+ # Default is 30 seconds
406
"VLLM_VIDEO_FETCH_TIMEOUT":
406
- lambda: int(os.getenv("VLLM_VIDEO_FETCH_TIMEOUT", "15")),
407
+ lambda: int(os.getenv("VLLM_VIDEO_FETCH_TIMEOUT", "30")),
408
409
# Timeout for fetching audio when serving multimodal models
410
# Default is 10 seconds
411
"VLLM_AUDIO_FETCH_TIMEOUT":
412
lambda: int(os.getenv("VLLM_AUDIO_FETCH_TIMEOUT", "10")),
413
414
+ # Cache size for multimodal feature/input cache for multimodal models
415
+ # in unit of number of multimodal data items (e.g. image, video, audio).
416
+ # Default is 256 multimodal data items.
417
+ "VLLM_MM_INPUT_CACHE_SIZE":
418
+ lambda: int(os.getenv("VLLM_MM_INPUT_CACHE_SIZE", "256")),
419
+
420
# Path to the XLA persistent cache directory.
421
# Only used for XLA devices such as TPUs.
422
"VLLM_XLA_CACHE_PATH":
vllm/multimodal/registry.py
+2
-4
@@ -8,6 +8,7 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, Mapping, Optional,
8
9
import torch.nn as nn
10
11
+from vllm.envs import VLLM_MM_INPUT_CACHE_SIZE
12
from vllm.inputs import InputProcessingContext
13
from vllm.logger import init_logger
14
from vllm.transformers_utils.tokenizer import AnyTokenizer
@@ -28,9 +29,6 @@ if TYPE_CHECKING:
29
30
logger = init_logger(__name__)
31
31
-# TODO: Tune the MM cache size
32
-MM_CACHE_SIZE = 256
33
-
32
N = TypeVar("N", bound=Type[nn.Module])
33
_I = TypeVar("_I", bound=BaseProcessingInfo)
34
_I_co = TypeVar("_I_co", bound=BaseProcessingInfo, covariant=True)
@@ -121,7 +119,7 @@ class MultiModalRegistry:
119
120
self._limits_by_model = _MultiModalLimits()
121
124
- self._processing_cache = ProcessingCache(MM_CACHE_SIZE)
122
+ self._processing_cache = ProcessingCache(VLLM_MM_INPUT_CACHE_SIZE)
123
124
def register_plugin(self, plugin: MultiModalPlugin) -> None:
125
"""
vllm/v1/engine/mm_input_cache.py
+7
-5
@@ -3,6 +3,7 @@
3
from typing import Any, Dict, List, Optional
4
5
from vllm.config import ModelConfig
6
+from vllm.envs import VLLM_MM_INPUT_CACHE_SIZE
7
from vllm.logger import init_logger
8
from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
9
MultiModalKwargs, MultiModalRegistry)
@@ -28,9 +29,8 @@ logger = init_logger(__name__)
29
# client (=P0) and server (=P1) processes.
30
31
# Both Client and Server must use the same cache size
31
-# (to perform mirrored caching)
32
-# TODO: Tune the MM cache size
33
-MM_CACHE_SIZE = 256
32
+# (to perform mirrored caching). This cache size is set by the environment
33
+# variable VLLM_MM_INPUT_CACHE_SIZE.
34
35
36
# TODO(ywang96): Deprecate this class once all multimodal models migrate to use
@@ -50,7 +50,8 @@ class MMInputCacheClient:
50
51
# Init cache
52
self.use_cache = not model_config.disable_mm_preprocessor_cache
53
- self.mm_cache = LRUCache[str, MultiModalKwargs](MM_CACHE_SIZE)
53
+ self.mm_cache = LRUCache[str,
54
+ MultiModalKwargs](VLLM_MM_INPUT_CACHE_SIZE)
55
56
# DEBUG: Set to None to disable
57
self.mm_debug_cache_hit_ratio_steps = None
@@ -127,7 +128,8 @@ class MMInputCacheServer:
128
129
def __init__(self, model_config):
130
self.use_cache = not model_config.disable_mm_preprocessor_cache
130
- self.mm_cache = LRUCache[str, MultiModalKwargs](MM_CACHE_SIZE)
131
+ self.mm_cache = LRUCache[str,
132
+ MultiModalKwargs](VLLM_MM_INPUT_CACHE_SIZE)
133
134
def get_and_update(
135
self,