A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Core][MM] Cleanup `MultiModalCache` (#25006)
Signed-off-by: Lukas Geiger <lukas.geiger94@gmail.com>
Lukas Geiger committed
Sep 18, 2025 at 05:08 UTC
b98219670fb1ca2952d449404c2b4921d7cdce73
1 file changed
+6
-19
vllm/multimodal/cache.py
+6
-19
@@ -1,5 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import operator
4
import sys
5
from abc import ABC, abstractmethod
6
from collections.abc import Mapping, Sequence
@@ -91,26 +92,15 @@ _V = TypeVar("_V", bound=MultiModalCacheValue)
92
class MultiModalCache:
93
94
@classmethod
94
- def get_leaf_size(
95
- cls,
96
- leaf: object,
97
- *,
98
- debug: bool = False,
99
- ) -> int:
95
+ def get_leaf_size(cls, leaf: object) -> int:
96
if isinstance(leaf, MultiModalProcessorCacheItem):
97
return cls.get_leaf_size(leaf.item)
98
if isinstance(leaf, MultiModalProcessorCacheItemMetadata):
99
return leaf.item_size
100
101
# These are not subclasses of dict
106
- if isinstance(leaf, MultiModalKwargsItems):
107
- return cls.get_item_size(leaf.data) # type: ignore
108
- if isinstance(leaf, MultiModalKwargsItem):
109
- return cls.get_item_size(leaf.data) # type: ignore
110
- if isinstance(leaf, MultiModalKwargs):
111
- return cls.get_item_size(leaf.data) # type: ignore
112
-
113
- if isinstance(leaf, MultiModalFieldElem):
102
+ if isinstance(leaf, (MultiModalKwargs, MultiModalKwargsItems,
103
+ MultiModalKwargsItem, MultiModalFieldElem)):
104
return cls.get_item_size(leaf.data) # type: ignore
105
106
# sys.getsizeof doesn't work for tensors
@@ -126,11 +116,8 @@ class MultiModalCache:
116
*,
117
debug: bool = False,
118
) -> int:
129
- size = json_reduce_leaves(
130
- lambda a, b: a + b,
131
- json_map_leaves(lambda x: cls.get_leaf_size(x, debug=debug),
132
- value),
133
- )
119
+ size = json_reduce_leaves(operator.add,
120
+ json_map_leaves(cls.get_leaf_size, value))
121
122
if debug:
123
leaf_count = json_count_leaves(value)