@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 139d155781

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Frontend] Use engine argument to control MM cache size (#22441)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Aug 8, 2025 at 00:47 UTC 139d155781c187b6d38ac6d84a516c97ff66bb1f

13 files changed +101 -47

docs/configuration/conserving_memory.md

+1 -1

@@ -86,7 +86,7 @@ llm = LLM(model="meta-llama/Llama-3.1-8B-Instruct",

87 If you run out of CPU RAM, try the following options:

89 -- (Multi-modal models only) you can set the size of multi-modal processor cache using `VLLM_MM_INPUT_CACHE_GIB` environment variable (default 4 GiB per API process + 4 GiB per engine core process)

89 +- (Multi-modal models only) you can set the size of multi-modal processor cache by setting `mm_processor_cache_gb` engine argument (default 4 GiB per API process + 4 GiB per engine core process)

90 - (CPU backend only) you can set the size of KV cache using `VLLM_CPU_KVCACHE_SPACE` environment variable (default 4 GiB).

92 ## Multi-modal input limits

docs/configuration/optimization.md

+9 -3

@@ -161,12 +161,18 @@ By default, the multi-modal processor cache is enabled to avoid repeatedly proce

161 the same multi-modal inputs via Hugging Face `AutoProcessor`,

162 which commonly occurs in multi-turn conversations.

163

164 -You can adjust the size of the cache via `VLLM_MM_INPUT_CACHE_GIB` environment variable

164 +You can adjust the size of the cache by setting the value of `mm_processor_cache_gb`

165 (default 4 GiB per API process + 4 GiB per engine core process).

166 +If you do not benefit much from the cache, you can disable it completely via `mm_processor_cache_gb=0`.

167

167 -If you do not benefit much from the cache, you can disable it completely via `disable_mm_preprocessor_cache`:

168 +Examples:

169

170 ```python

171 +# Use a larger cache

172 llm = LLM(model="Qwen/Qwen2.5-VL-3B-Instruct",

171 - disable_mm_preprocessor_cache=True)

173 + mm_processor_cache_gb=8)

174 +

175 +# Disable the cache

176 +llm = LLM(model="Qwen/Qwen2.5-VL-3B-Instruct",

177 + mm_processor_cache_gb=0)

178 ```

examples/offline_inference/mistral-small.py

+3 -3

@@ -68,7 +68,7 @@ def run_simple_demo(args: argparse.Namespace):

68 max_model_len=4096,

69 max_num_seqs=2,

70 tensor_parallel_size=2,

71 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

71 + mm_processor_cache_gb=0 if args.disable_mm_processor_cache else 4,

72 )

74 prompt = "Describe this image in one sentence."

@@ -105,7 +105,7 @@ def run_advanced_demo(args: argparse.Namespace):

105 limit_mm_per_prompt={"image": max_img_per_msg},

106 max_model_len=max_img_per_msg * max_tokens_per_img,

107 tensor_parallel_size=2,

108 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

108 + mm_processor_cache_gb=0 if args.disable_mm_processor_cache else 4,

109 )

110

111 prompt = "Describe the following image."

@@ -164,7 +164,7 @@ def parse_args():

164 )

165

166 parser.add_argument(

167 - "--disable-mm-preprocessor-cache",

167 + "--disable-mm-processor-cache",

168 action="store_true",

169 help="If True, disables caching of multi-modal processor.",

170 )

examples/offline_inference/vision_language.py

+2 -2

@@ -1563,7 +1563,7 @@ def parse_args():

1563 )

1564

1565 parser.add_argument(

1566 - "--disable-mm-preprocessor-cache",

1566 + "--disable-mm-processor-cache",

1567 action="store_true",

1568 help="If True, disables caching of multi-modal processor.",

1569 )

@@ -1603,7 +1603,7 @@ def main(args):

1603

1604 engine_args = asdict(req_data.engine_args) | {

1605 "seed": args.seed,

1606 - "disable_mm_preprocessor_cache": args.disable_mm_preprocessor_cache,

1606 + "mm_processor_cache_gb": 0 if args.disable_mm_processor_cache else 4,

1607 }

1608 llm = LLM(**engine_args)

1609

tests/models/multimodal/generation/vlm_utils/core.py

+1 -3

@@ -62,9 +62,7 @@ def run_test(

62 # if we run HF first, the cuda initialization will be done and it

63 # will hurt multiprocessing backend with fork method (the default method).

65 - vllm_runner_kwargs_: dict[str, Any] = {

66 - "disable_mm_preprocessor_cache": True,

67 - }

65 + vllm_runner_kwargs_: dict[str, Any] = {"mm_processor_cache_gb": 0}

66 if model_info.tokenizer:

67 vllm_runner_kwargs_["tokenizer_name"] = model_info.tokenizer

68 if model_info.tokenizer_mode:

tests/models/multimodal/processing/test_llama4.py

+3 -3

@@ -15,14 +15,14 @@ from ...utils import build_model_context

15 ["meta-llama/Llama-4-Scout-17B-16E-Instruct"])

16 @pytest.mark.parametrize("mm_processor_kwargs", [{}])

17 @pytest.mark.parametrize("num_imgs", [1, 5])

18 -@pytest.mark.parametrize("disable_mm_preprocessor_cache", [True, False])

18 +@pytest.mark.parametrize("mm_processor_cache_gb", [0, 4])

19 @pytest.mark.parametrize("tokenized_prompt", [True, False])

20 def test_processor_override(

21 image_assets: ImageTestAssets,

22 model_id: str,

23 mm_processor_kwargs: dict,

24 num_imgs: int,

25 - disable_mm_preprocessor_cache: bool,

25 + mm_processor_cache_gb: int,

26 tokenized_prompt: bool,

27 ):

28 """Ensure llama4 processor works properly."""

@@ -30,7 +30,7 @@ def test_processor_override(

30 model_id,

31 mm_processor_kwargs=mm_processor_kwargs,

32 limit_mm_per_prompt={"image": num_imgs},

33 - disable_mm_preprocessor_cache=disable_mm_preprocessor_cache,

33 + mm_processor_cache_gb=mm_processor_cache_gb,

34 )

35 processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

36 config = processor.info.get_hf_config()

tests/models/utils.py

+2 -2

@@ -261,7 +261,7 @@ def build_model_context(

261 model_config_kwargs: Optional[dict[str, Any]] = None,

262 mm_processor_kwargs: Optional[dict[str, Any]] = None,

263 limit_mm_per_prompt: Optional[dict[str, int]] = None,

264 - disable_mm_preprocessor_cache: bool = True,

264 + mm_processor_cache_gb: int = 0,

265 ):

266 """Creates an InputContext for a given model.

267

@@ -291,7 +291,7 @@ def build_model_context(

291 seed=0,

292 mm_processor_kwargs=mm_processor_kwargs,

293 limit_mm_per_prompt=limit_mm_per_prompt,

294 - disable_mm_preprocessor_cache=disable_mm_preprocessor_cache,

294 + mm_processor_cache_gb=mm_processor_cache_gb,

295 hf_overrides=model_info.hf_overrides,

296 **model_config_kwargs,

297 )

vllm/config.py

+32 -11

@@ -443,8 +443,15 @@ class ModelConfig:

443 from `AutoProcessor.from_pretrained`. The available overrides depend on the

444 model that is being run. For example, for Phi-3-Vision: `{"num_crops": 4}`.

445 """

446 - disable_mm_preprocessor_cache: bool = False

447 - """If `True`, disable caching of the multi-modal processor."""

446 + mm_processor_cache_gb: int = 4

447 + """The size (in GiB) of the multi-modal processor cache, which is used to

448 + avoid re-processing past multi-modal inputs.

449 +

450 + This cache is duplicated for each API process and engine core process,

451 + resulting in a total memory usage of

452 + `mm_processor_cache_gb * (api_server_count + data_parallel_size)`.

453 +

454 + Set to `0` to disable this cache completely (not recommended)."""

455 override_neuron_config: dict[str, Any] = field(default_factory=dict)

456 """Initialize non-default neuron config or override default neuron config

457 that are specific to Neuron devices, this argument will be used to

@@ -881,17 +888,16 @@ class ModelConfig:

888 limit_per_prompt=self.limit_mm_per_prompt,

889 media_io_kwargs=self.media_io_kwargs,

890 mm_processor_kwargs=self.mm_processor_kwargs,

884 - disable_mm_preprocessor_cache=self.

885 - disable_mm_preprocessor_cache,

891 + mm_processor_cache_gb=self.mm_processor_cache_gb,

892 interleave_mm_strings=self.interleave_mm_strings)

893

894 return None

895

890 - def set_disable_mm_preprocessor_cache(self, value: bool) -> None:

896 + def set_mm_processor_cache_gb(self, value: int) -> None:

897 mm_config = self.get_multimodal_config()

898

893 - self.disable_mm_preprocessor_cache = value

894 - mm_config.disable_mm_preprocessor_cache = value

899 + self.mm_processor_cache_gb = value

900 + mm_config.mm_processor_cache_gb = value

901

902 def _get_encoder_config(self):

903 return get_sentence_transformer_tokenizer_config(

@@ -1698,7 +1704,16 @@ class ModelConfig:

1704 if mm_config is None:

1705 return False

1706

1701 - return not mm_config.disable_mm_preprocessor_cache

1707 + return mm_config.mm_processor_cache_gb > 0

1708 +

1709 + @property

1710 + def enable_mm_processor_cache(self) -> bool:

1711 + """Whether the multi-modal processor cache should be enabled."""

1712 + mm_config = self.multimodal_config

1713 + if mm_config is None:

1714 + return False

1715 +

1716 + return mm_config.mm_processor_cache_gb > 0

1717

1718 @property

1719 def enable_mm_input_cache(self) -> bool:

@@ -1707,7 +1722,7 @@ class ModelConfig:

1722 if mm_config is None:

1723 return False

1724

1710 - return not mm_config.disable_mm_preprocessor_cache

1725 + return mm_config.mm_processor_cache_gb > 0

1726

1727 def get_mm_input_cache_gb(self) -> int:

1728 mm_config = self.multimodal_config

@@ -3391,9 +3406,15 @@ class MultiModalConfig:

3406 `{"num_crops": 4}`.

3407 """

3408

3394 - disable_mm_preprocessor_cache: bool = False

3409 + mm_processor_cache_gb: int = 4

3410 """

3396 - If `True`, disable caching of the multi-modal processor.

3411 + The size (in GiB) of the multi-modal processor cache, which is used to

3412 +

3413 + This cache is duplicated for each API process and engine core process,

3414 + resulting in a total memory usage of

3415 + `mm_processor_cache_gb * (api_server_count + data_parallel_size)`.

3416 +

3417 + Set to `0` to disable this cache completely (not recommended).

3418 """

3419

3420 interleave_mm_strings: bool = False

vllm/engine/arg_utils.py

+27 -7

@@ -358,8 +358,8 @@ class EngineArgs:

358 "media_io_kwargs")

359 mm_processor_kwargs: Optional[Dict[str, Any]] = \

360 MultiModalConfig.mm_processor_kwargs

361 - disable_mm_preprocessor_cache: bool = \

362 - MultiModalConfig.disable_mm_preprocessor_cache

361 + disable_mm_preprocessor_cache: bool = False # DEPRECATED

362 + mm_processor_cache_gb: int = MultiModalConfig.mm_processor_cache_gb

363 # LoRA fields

364 enable_lora: bool = False

365 enable_lora_bias: bool = LoRAConfig.bias_enabled

@@ -720,8 +720,11 @@ class EngineArgs:

720 "--mm-processor-kwargs",

721 **multimodal_kwargs["mm_processor_kwargs"])

722 multimodal_group.add_argument(

723 - "--disable-mm-preprocessor-cache",

724 - **multimodal_kwargs["disable_mm_preprocessor_cache"])

723 + "--mm-processor-cache-gb",

724 + **multimodal_kwargs["mm_processor_cache_gb"])

725 + multimodal_group.add_argument("--disable-mm-preprocessor-cache",

726 + type=bool,

727 + deprecated=True)

728 multimodal_group.add_argument(

729 "--interleave-mm-strings",

730 **multimodal_kwargs["interleave_mm_strings"])

@@ -886,6 +889,23 @@ class EngineArgs:

889 self.model = f"{MODEL_WEIGHTS_S3_BUCKET}/{self.model}"

890 self.load_format = "runai_streamer"

891

892 + if self.disable_mm_preprocessor_cache:

893 + logger.warning(

894 + "`--disable-mm-preprocessor-cache` is deprecated "

895 + "and will be removed in v0.13. "

896 + "Please use `--mm-processor-cache-gb 0` instead.", )

897 +

898 + self.mm_processor_cache_gb = 0

899 + elif envs.VLLM_MM_INPUT_CACHE_GIB != 4:

900 + logger.warning(

901 + "VLLM_MM_INPUT_CACHE_GIB` is deprecated "

902 + "and will be removed in v0.13. "

903 + "Please use `--mm-processor-cache-gb %d` instead.",

904 + envs.VLLM_MM_INPUT_CACHE_GIB,

905 + )

906 +

907 + self.mm_processor_cache_gb = envs.VLLM_MM_INPUT_CACHE_GIB

908 +

909 return ModelConfig(

910 model=self.model,

911 hf_config_path=self.hf_config_path,

@@ -922,7 +942,7 @@ class EngineArgs:

942 use_async_output_proc=not self.disable_async_output_proc,

943 config_format=self.config_format,

944 mm_processor_kwargs=self.mm_processor_kwargs,

925 - disable_mm_preprocessor_cache=self.disable_mm_preprocessor_cache,

945 + mm_processor_cache_gb=self.mm_processor_cache_gb,

946 override_neuron_config=self.override_neuron_config,

947 override_pooler_config=self.override_pooler_config,

948 logits_processor_pattern=self.logits_processor_pattern,

@@ -1234,13 +1254,13 @@ class EngineArgs:

1254 dp_supports_mm_processor_cache = (self.data_parallel_size == 1

1255 or data_parallel_external_lb)

1256 if (not dp_supports_mm_processor_cache

1237 - and not model_config.disable_mm_preprocessor_cache):

1257 + and model_config.mm_processor_cache_gb > 0):

1258 logger.warning(

1259 "Multi-modal processor cache is disabled because "

1260 "it is not compatible with data parallelism when "

1261 "there does not exist a one-to-one correspondance "

1262 "between API and engine core processes.")

1243 - model_config.set_disable_mm_preprocessor_cache(True)

1263 + model_config.set_mm_processor_cache_gb(0)

1264

1265 speculative_config = self.create_speculative_config(

1266 target_model_config=model_config,

vllm/entrypoints/cli/serve.py

+3 -4

@@ -138,13 +138,13 @@ def run_multi_api_server(args: argparse.Namespace):

138 num_api_servers = args.api_server_count

139 assert num_api_servers > 0

140

141 - orig_disable_mm_preprocessor_cache = args.disable_mm_preprocessor_cache

141 + orig_mm_processor_cache_gb = args.mm_processor_cache_gb

142

143 if num_api_servers > 1:

144 setup_multiprocess_prometheus()

145

146 # Not compatible with API server scale-out

147 - args.disable_mm_preprocessor_cache = True

147 + args.mm_processor_cache_gb = 0

148

149 listen_address, sock = setup_server(args)

150

@@ -161,8 +161,7 @@ def run_multi_api_server(args: argparse.Namespace):

161 raise ValueError("VLLM_ALLOW_RUNTIME_LORA_UPDATING cannot be used "

162 "with api_server_count > 1")

163

164 - if model_config.is_multimodal_model and not (

165 - orig_disable_mm_preprocessor_cache):

164 + if model_config.is_multimodal_model and orig_mm_processor_cache_gb > 0:

165 logger.warning("Multi-modal processor cache is disabled because "

166 "it is not compatible with `api_server_count > 1`.")

167

vllm/envs.py

+1 -1

@@ -561,7 +561,7 @@ environment_variables: dict[str, Callable[[], Any]] = {

561 "VLLM_VIDEO_LOADER_BACKEND":

562 lambda: os.getenv("VLLM_VIDEO_LOADER_BACKEND", "opencv"),

563

564 - # Cache size (in GiB per process) for multimodal input cache

564 + # [DEPRECATED] Cache size (in GiB per process) for multimodal input cache

565 # Default is 4 GiB per API process + 4 GiB per engine core process

566 "VLLM_MM_INPUT_CACHE_GIB":

567 lambda: int(os.getenv("VLLM_MM_INPUT_CACHE_GIB", "4")),

vllm/multimodal/registry.py

+16 -6

@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Generic, Optional, Protocol, TypeVar

7 import torch.nn as nn

9 -from vllm.envs import VLLM_MM_INPUT_CACHE_GIB

9 from vllm.inputs import InputProcessingContext

10 from vllm.logger import init_logger

11 from vllm.transformers_utils.tokenizer import (AnyTokenizer,

@@ -96,11 +95,22 @@ class MultiModalRegistry:

95 self._processor_factories = ClassRegistry[nn.Module,

96 _ProcessorFactories]()

99 - self._processing_cache = ProcessingCache(VLLM_MM_INPUT_CACHE_GIB)

98 + self._processor_cache: Optional[ProcessingCache] = None

99 +

100 + def _get_processor_cache(self, model_config: "ModelConfig"):

101 + capacity_gb = model_config.mm_processor_cache_gb

102 + if capacity_gb is None:

103 + return None # Overrides `disable_cache` argument

104 +

105 + if self._processor_cache is None:

106 + self._processor_cache = ProcessingCache(capacity_gb)

107 +

108 + return self._processor_cache

109

110 def reset_processor_cache(self) -> bool:

111 """Reset the multi-modal processing cache."""

103 - self._processing_cache.reset()

112 + if self._processor_cache:

113 + self._processor_cache.reset()

114

115 return True # Success

116

@@ -244,14 +254,14 @@ class MultiModalRegistry:

254 if tokenizer is None and not model_config.skip_tokenizer_init:

255 tokenizer = cached_tokenizer_from_config(model_config)

256 if disable_cache is None:

247 - mm_config = model_config.get_multimodal_config()

248 - disable_cache = mm_config.disable_mm_preprocessor_cache

257 + disable_cache = not model_config.enable_mm_processor_cache

258

259 model_cls = self._get_model_cls(model_config)

260 factories = self._processor_factories[model_cls]

261

262 ctx = InputProcessingContext(model_config, tokenizer)

254 - cache = None if disable_cache else self._processing_cache

263 + cache = None if disable_cache else self._get_processor_cache(

264 + model_config)

265

266 return factories.build_processor(ctx, cache=cache)

267

vllm/v1/core/kv_cache_utils.py

+1 -1

@@ -430,7 +430,7 @@ def _gen_mm_extra_hash_keys(request: Request, start_token_idx: int,

430 raise ValueError(

431 "The number of multi-modal positions and hashes must match. This "

432 "is likely because you did not enable MM hashing. "

433 - "Please set `disable_mm_preprocessor_cache=False`.")

433 + "Please set `mm_processor_cache_gb > 0`.")

434

435 # Note that we assume mm_positions is sorted by offset.

436 # We do not need to check all mm inputs if the start token index is out of