@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dd5ede4440
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[V1] Consolidate MM cache size to vllm.envs (#13239)

Roger Wang committed Feb 13, 2025 at 20:19 UTC dd5ede444032cfb00d8377722b7f24ec9157666b
3 files changed +18 -11
vllm/envs.py
+9 -2
@@ -55,6 +55,7 @@ if TYPE_CHECKING:
55 VLLM_IMAGE_FETCH_TIMEOUT: int = 5
56 VLLM_VIDEO_FETCH_TIMEOUT: int = 30
57 VLLM_AUDIO_FETCH_TIMEOUT: int = 10
58 + VLLM_MM_INPUT_CACHE_SIZE: int = 256
59 VLLM_TARGET_DEVICE: str = "cuda"
60 MAX_JOBS: Optional[str] = None
61 NVCC_THREADS: Optional[str] = None
@@ -401,15 +402,21 @@ environment_variables: Dict[str, Callable[[], Any]] = {
402 lambda: int(os.getenv("VLLM_IMAGE_FETCH_TIMEOUT", "5")),
403
404 # Timeout for fetching videos when serving multimodal models
404 - # Default is 15 seconds
405 + # Default is 30 seconds
406 "VLLM_VIDEO_FETCH_TIMEOUT":
406 - lambda: int(os.getenv("VLLM_VIDEO_FETCH_TIMEOUT", "15")),
407 + lambda: int(os.getenv("VLLM_VIDEO_FETCH_TIMEOUT", "30")),
408
409 # Timeout for fetching audio when serving multimodal models
410 # Default is 10 seconds
411 "VLLM_AUDIO_FETCH_TIMEOUT":
412 lambda: int(os.getenv("VLLM_AUDIO_FETCH_TIMEOUT", "10")),
413
414 + # Cache size for multimodal feature/input cache for multimodal models
415 + # in unit of number of multimodal data items (e.g. image, video, audio).
416 + # Default is 256 multimodal data items.
417 + "VLLM_MM_INPUT_CACHE_SIZE":
418 + lambda: int(os.getenv("VLLM_MM_INPUT_CACHE_SIZE", "256")),
419 +
420 # Path to the XLA persistent cache directory.
421 # Only used for XLA devices such as TPUs.
422 "VLLM_XLA_CACHE_PATH":
vllm/multimodal/registry.py
+2 -4
@@ -8,6 +8,7 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, Mapping, Optional,
8
9 import torch.nn as nn
10
11 +from vllm.envs import VLLM_MM_INPUT_CACHE_SIZE
12 from vllm.inputs import InputProcessingContext
13 from vllm.logger import init_logger
14 from vllm.transformers_utils.tokenizer import AnyTokenizer
@@ -28,9 +29,6 @@ if TYPE_CHECKING:
29
30 logger = init_logger(__name__)
31
31 -# TODO: Tune the MM cache size
32 -MM_CACHE_SIZE = 256
33 -
32 N = TypeVar("N", bound=Type[nn.Module])
33 _I = TypeVar("_I", bound=BaseProcessingInfo)
34 _I_co = TypeVar("_I_co", bound=BaseProcessingInfo, covariant=True)
@@ -121,7 +119,7 @@ class MultiModalRegistry:
119
120 self._limits_by_model = _MultiModalLimits()
121
124 - self._processing_cache = ProcessingCache(MM_CACHE_SIZE)
122 + self._processing_cache = ProcessingCache(VLLM_MM_INPUT_CACHE_SIZE)
123
124 def register_plugin(self, plugin: MultiModalPlugin) -> None:
125 """
vllm/v1/engine/mm_input_cache.py
+7 -5
@@ -3,6 +3,7 @@
3 from typing import Any, Dict, List, Optional
4
5 from vllm.config import ModelConfig
6 +from vllm.envs import VLLM_MM_INPUT_CACHE_SIZE
7 from vllm.logger import init_logger
8 from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
9 MultiModalKwargs, MultiModalRegistry)
@@ -28,9 +29,8 @@ logger = init_logger(__name__)
29 # client (=P0) and server (=P1) processes.
30
31 # Both Client and Server must use the same cache size
31 -# (to perform mirrored caching)
32 -# TODO: Tune the MM cache size
33 -MM_CACHE_SIZE = 256
32 +# (to perform mirrored caching). This cache size is set by the environment
33 +# variable VLLM_MM_INPUT_CACHE_SIZE.
34
35
36 # TODO(ywang96): Deprecate this class once all multimodal models migrate to use
@@ -50,7 +50,8 @@ class MMInputCacheClient:
50
51 # Init cache
52 self.use_cache = not model_config.disable_mm_preprocessor_cache
53 - self.mm_cache = LRUCache[str, MultiModalKwargs](MM_CACHE_SIZE)
53 + self.mm_cache = LRUCache[str,
54 + MultiModalKwargs](VLLM_MM_INPUT_CACHE_SIZE)
55
56 # DEBUG: Set to None to disable
57 self.mm_debug_cache_hit_ratio_steps = None
@@ -127,7 +128,8 @@ class MMInputCacheServer:
128
129 def __init__(self, model_config):
130 self.use_cache = not model_config.disable_mm_preprocessor_cache
130 - self.mm_cache = LRUCache[str, MultiModalKwargs](MM_CACHE_SIZE)
131 + self.mm_cache = LRUCache[str,
132 + MultiModalKwargs](VLLM_MM_INPUT_CACHE_SIZE)
133
134 def get_and_update(
135 self,