@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 1180b604f8

[Multimodal] Expose mm hash algothrim selection to cli args (#49686)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>

Isotr0py committed Jul 31, 2026 at 13:25 UTC 1180b604f8d5559a00d17b8f384ddcdbc9a41d16

17 files changed +234 -61

docs/usage/security.md

+8 -2

@@ -396,7 +396,7 @@ FIPS compliance depends on many factors, so a vLLM deployment is not automatical

396

397 Operators running vLLM on FIPS-enabled hosts should select FIPS-approved algorithms via the following knobs:

398

399 -- **Multimodal input hashing** — `VLLM_MM_HASHER_ALGORITHM` defaults to `blake3`, which is not FIPS-approved. Set it to `sha256` or `sha512` in FIPS-enabled environments.

399 +- **Multimodal input hashing** — `--mm-hasher-algorithm` (config field `mm_hasher_algorithm`) defaults to `blake3`, which is not FIPS-approved. Set it to `sha256` or `sha512` in FIPS-enabled environments.

400 - **Prefix-cache hashing** — set `--prefix-caching-hash-algo` (config field `prefix_caching_hash_algo`) to `sha256` or `sha256_cbor`. The `xxhash` and `xxhash_cbor` options are not FIPS-approved.

401 - **TLS ciphers** — use `--ssl-ciphers` to restrict the API server's TLS handshake to FIPS-approved cipher suites that match your environment's policy.

402

@@ -408,7 +408,13 @@ vLLM uses MD5 in a few places to derive non-security cache keys (for example, co

408

409 Some dependencies expose hash implementations that are not FIPS-approved. vLLM only invokes them when the corresponding algorithm is selected, but operators with strict cryptographic controls may want to ensure the code paths are not exercised — and, where policy requires, that the packages themselves are absent:

410

411 -- `blake3` — currently listed in `requirements/common.txt`, so a standard install pulls it in. It is imported lazily and only used when `VLLM_MM_HASHER_ALGORITHM=blake3` (the default). Setting `VLLM_MM_HASHER_ALGORITHM` to `sha256` or `sha512` is sufficient to keep the non-FIPS code path dormant. If your policy additionally forbids the package being present, uninstall it after `pip install` (`pip uninstall blake3`); vLLM will continue to function as long as `VLLM_MM_HASHER_ALGORITHM` is set to a non-blake3 value.

411 +- `blake3` — currently listed in `requirements/common.txt`, so a standard

412 + install pulls it in. It is imported lazily and only used when

413 + `mm_hasher_algorithm=blake3` (the default). Setting

414 + `--mm-hasher-algorithm sha256` or `--mm-hasher-algorithm sha512` is sufficient

415 + to keep the non-FIPS code path dormant. If your policy additionally forbids

416 + the package being present, uninstall it after installation; vLLM will

417 + continue to function as long as a non-blake3 algorithm is selected.

418 - `xxhash` — a true optional dependency (not in `requirements/common.txt`). It is only imported when an `xxhash`-based prefix-cache algorithm is selected. Leave it uninstalled and select a `sha256`-based prefix-cache algorithm.

419

420 ### Beyond hashing: other FIPS considerations

tests/config/test_multimodal_config.py

@@ -24,6 +24,11 @@ def test_mm_encoder_attn_backend_invalid():

24 MultiModalConfig(mm_encoder_attn_backend="not_a_backend") # type: ignore[arg-type]

27 +def test_mm_hasher_algorithm_invalid():

28 + with pytest.raises(ValueError, match="mm_hasher_algorithm"):

29 + MultiModalConfig(mm_hasher_algorithm="md5") # type: ignore[arg-type]

30 +

31 +

32 def test_mm_encoder_attn_backend_hash_updates():

33 base_hash = MultiModalConfig().compute_hash()

34 overridden_hash = MultiModalConfig(

tests/models/multimodal/processing/test_moss_audio.py

@@ -50,6 +50,7 @@ class _Tokenizer:

50 class _MMConfig:

51 enable_mm_embeds = False

52 mm_processor_cache_gb = 1

53 + mm_hasher_algorithm = "blake3"

55 def merge_mm_processor_kwargs(self, kwargs):

56 return dict(kwargs)

tests/multimodal/test_cache.py

+2 -1

@@ -144,7 +144,8 @@ def _compare_caches(

144 for _ in range(int(item_capacity / hit_rate))

145 ]

146 all_hashes = [

147 - MultiModalHasher.hash_kwargs(item=item.get_data()) for item in all_items

147 + MultiModalHasher.hash_kwargs("blake3", item=item.get_data())

148 + for item in all_items

149 ]

150

151 prompt_update = PromptInsertion("dummy", "target", "insertion").resolve(0)

tests/multimodal/test_hasher.py

+58 -16

@@ -1,5 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +import hashlib

4 import uuid

5 from io import BytesIO

6 from pathlib import Path

@@ -9,6 +10,7 @@ import pytest

10 import torch

11 from PIL import Image, ImageDraw

13 +from vllm.config.multimodal import MMHasherAlgorithm

14 from vllm.multimodal.hasher import MultiModalHasher

15 from vllm.multimodal.media.base import MediaWithBytes

16 from vllm.multimodal.media.image import ImageMediaIO

@@ -20,12 +22,26 @@ ASSETS_DIR = Path(__file__).parent / "assets"

22 assert ASSETS_DIR.exists()

25 +@pytest.mark.parametrize("algorithm", ["sha256", "sha512"])

26 +def test_hash_algorithm(algorithm: MMHasherAlgorithm):

27 + hasher = getattr(hashlib, algorithm)()

28 + for bytes_ in MultiModalHasher.iter_item_to_bytes("value", "test"):

29 + hasher.update(bytes_)

30 +

31 + assert MultiModalHasher.hash_kwargs(algorithm, value="test") == hasher.hexdigest()

32 +

33 +

34 +def test_hash_algorithm_required():

35 + with pytest.raises(TypeError, match="algorithm"):

36 + MultiModalHasher.hash_kwargs(value="test") # type: ignore[call-arg]

37 +

38 +

39 def test_hash_single_item_different_shape():

40 x1 = torch.zeros(())

41 x2 = torch.zeros((1,))

43 hasher = MultiModalHasher

28 - assert hasher.hash_kwargs(x=x1) != hasher.hash_kwargs(x=x2)

44 + assert hasher.hash_kwargs("blake3", x=x1) != hasher.hash_kwargs("blake3", x=x2)

47 def test_hash_key_order_invariant():

@@ -33,7 +49,9 @@ def test_hash_key_order_invariant():

49 y = torch.ones((5, 10))

51 hasher = MultiModalHasher

36 - assert hasher.hash_kwargs(x=x, y=y) == hasher.hash_kwargs(y=y, x=x)

52 + assert hasher.hash_kwargs("blake3", x=x, y=y) == hasher.hash_kwargs(

53 + "blake3", y=y, x=x

54 + )

57 # NOTE: Images that are the same visually are allowed to have the same hash

@@ -44,7 +62,9 @@ def test_hash_collision_image_mode(mode_pair):

62 image2 = Image.new(mode2, size=(10, 10), color=1)

64 hasher = MultiModalHasher

47 - assert hasher.hash_kwargs(image=image1) != hasher.hash_kwargs(image=image2)

65 + assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs(

66 + "blake3", image=image2

67 + )

70 def test_hash_collision_image_palette():

@@ -53,7 +73,9 @@ def test_hash_collision_image_palette():

73 image2 = Image.open(ASSETS_DIR / "image2.png")

75 hasher = MultiModalHasher

56 - assert hasher.hash_kwargs(image=image1) != hasher.hash_kwargs(image=image2)

76 + assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs(

77 + "blake3", image=image2

78 + )

81 def test_hash_collision_image_transpose():

@@ -64,7 +86,9 @@ def test_hash_collision_image_transpose():

86 ImageDraw.Draw(image2).line([(0, 0), (0, 10)])

88 hasher = MultiModalHasher

67 - assert hasher.hash_kwargs(image=image1) != hasher.hash_kwargs(image=image2)

89 + assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs(

90 + "blake3", image=image2

91 + )

94 @pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])

@@ -74,7 +98,9 @@ def test_hash_collision_tensor_shape(dtype):

98 arr2 = torch.zeros((10, 20, 5, 3), dtype=dtype)

100 hasher = MultiModalHasher

77 - assert hasher.hash_kwargs(data=arr1) != hasher.hash_kwargs(data=arr2)

101 + assert hasher.hash_kwargs("blake3", data=arr1) != hasher.hash_kwargs(

102 + "blake3", data=arr2

103 + )

104

105

106 def test_hash_collision_array_shape():

@@ -83,7 +109,9 @@ def test_hash_collision_array_shape():

109 arr2 = np.zeros((10, 20, 5, 3))

110

111 hasher = MultiModalHasher

86 - assert hasher.hash_kwargs(data=arr1) != hasher.hash_kwargs(data=arr2)

112 + assert hasher.hash_kwargs("blake3", data=arr1) != hasher.hash_kwargs(

113 + "blake3", data=arr2

114 + )

115

116

117 def test_hash_collision_video_num_frames():

@@ -104,8 +132,8 @@ def test_hash_collision_video_num_frames():

132 return items.get_all_items_for_hash()[0]

133

134 hasher = MultiModalHasher

107 - assert hasher.hash_kwargs(video=item_for_hash(2)) != hasher.hash_kwargs(

108 - video=item_for_hash(4)

135 + assert hasher.hash_kwargs("blake3", video=item_for_hash(2)) != hasher.hash_kwargs(

136 + "blake3", video=item_for_hash(4)

137 )

138

139

@@ -118,7 +146,9 @@ def test_hash_non_contiguous_array():

146

147 hasher = MultiModalHasher

148 # Both should be hashable and produce the same hashes

121 - assert hasher.hash_kwargs(data=arr) == hasher.hash_kwargs(data=arr_c)

149 + assert hasher.hash_kwargs("blake3", data=arr) == hasher.hash_kwargs(

150 + "blake3", data=arr_c

151 + )

152

153

154 def test_hash_image_exif_id():

@@ -133,9 +163,13 @@ def test_hash_image_exif_id():

163

164 hasher = MultiModalHasher

165 # first image has UUID in ImageID, so it should hash to that UUID

136 - assert hasher.hash_kwargs(image=image1) == hasher.hash_kwargs(image=id.bytes)

166 + assert hasher.hash_kwargs("blake3", image=image1) == hasher.hash_kwargs(

167 + "blake3", image=id.bytes

168 + )

169 # second image has non-UUID in ImageID, so it should hash to the image data

138 - assert hasher.hash_kwargs(image=image2) == hasher.hash_kwargs(image=image2a)

170 + assert hasher.hash_kwargs("blake3", image=image2) == hasher.hash_kwargs(

171 + "blake3", image=image2a

172 + )

173

174

175 def _rgba_png_bytes() -> bytes:

@@ -153,9 +187,15 @@ def test_hash_collision_media_io_config():

187 keep = ImageMediaIO(image_mode=None).load_bytes(data)

188

189 hasher = MultiModalHasher

156 - assert hasher.hash_kwargs(image=white) != hasher.hash_kwargs(image=black)

157 - assert hasher.hash_kwargs(image=white) != hasher.hash_kwargs(image=keep)

158 - assert hasher.hash_kwargs(image=white) == hasher.hash_kwargs(image=white2)

190 + assert hasher.hash_kwargs("blake3", image=white) != hasher.hash_kwargs(

191 + "blake3", image=black

192 + )

193 + assert hasher.hash_kwargs("blake3", image=white) != hasher.hash_kwargs(

194 + "blake3", image=keep

195 + )

196 + assert hasher.hash_kwargs("blake3", image=white) == hasher.hash_kwargs(

197 + "blake3", image=white2

198 + )

199

200

201 def test_hash_media_io_noop_config_preserves_hash():

@@ -169,4 +209,6 @@ def test_hash_media_io_noop_config_preserves_hash():

209

210 plain = MediaWithBytes(loaded.media, data)

211 hasher = MultiModalHasher

172 - assert hasher.hash_kwargs(image=loaded) == hasher.hash_kwargs(image=plain)

212 + assert hasher.hash_kwargs("blake3", image=loaded) == hasher.hash_kwargs(

213 + "blake3", image=plain

214 + )

tests/multimodal/test_utils.py

+25

@@ -199,6 +199,31 @@ def test_group_and_batch_mm_items_split_by_fieldset():

199 assert [num_items for num_items, _ in res] == [2, 1, 1, 1]

200

201

202 +def test_group_and_batch_mm_items_splits_shared_data_by_dtype():

203 + elem1 = MultiModalFieldElem(

204 + data=torch.zeros(1, dtype=torch.int32),

205 + field=MultiModalSharedField(batch_size=1),

206 + )

207 + elem2 = MultiModalFieldElem(

208 + data=torch.zeros(1, dtype=torch.float32),

209 + field=MultiModalSharedField(batch_size=1),

210 + )

211 + elem3 = MultiModalFieldElem(

212 + data=[torch.zeros(1, dtype=torch.int32), torch.zeros(1, dtype=torch.float32)],

213 + field=MultiModalSharedField(batch_size=1),

214 + )

215 +

216 + res = group_and_batch_mm_items(

217 + [

218 + MultiModalKwargsItem({"x": elem1}),

219 + MultiModalKwargsItem({"x": elem2}),

220 + MultiModalKwargsItem({"x": elem3}),

221 + ]

222 + )

223 +

224 + assert [num_items for num_items, _ in res] == [1, 1, 1]

225 +

226 +

227 def test_group_and_batch_mm_items_split_by_shared_data():

228 elem1 = MultiModalFieldElem(

229 data=torch.zeros(1, dtype=torch.uint8),

vllm/config/model.py

@@ -17,6 +17,7 @@ from vllm.config.model_arch import (

17 from vllm.config.multimodal import (

18 MMCacheType,

19 MMEncoderTPMode,

20 + MMHasherAlgorithm,

21 MMTensorIPC,

22 MultiModalConfig,

23 )

@@ -374,6 +375,7 @@ class ModelConfig:

375 mm_processor_kwargs: InitVar[dict[str, Any] | None] = None

376 mm_processor_cache_gb: InitVar[float | None] = None

377 mm_processor_cache_type: InitVar[MMCacheType | None] = None

378 + mm_hasher_algorithm: InitVar[MMHasherAlgorithm | None] = None

379 mm_shm_cache_max_object_size_mb: InitVar[int | None] = None

380 mm_encoder_only: InitVar[bool | None] = None

381 mm_encoder_tp_mode: InitVar[MMEncoderTPMode | None] = None

@@ -503,6 +505,7 @@ class ModelConfig:

505 mm_processor_kwargs: dict[str, Any] | None,

506 mm_processor_cache_gb: float | None,

507 mm_processor_cache_type: MMCacheType | None,

508 + mm_hasher_algorithm: MMHasherAlgorithm | None,

509 mm_shm_cache_max_object_size_mb: int | None,

510 mm_encoder_only: bool | None,

511 mm_encoder_tp_mode: MMEncoderTPMode | None,

@@ -737,6 +740,7 @@ class ModelConfig:

740 mm_processor_kwargs=mm_processor_kwargs,

741 mm_processor_cache_gb=mm_processor_cache_gb,

742 mm_processor_cache_type=mm_processor_cache_type,

743 + mm_hasher_algorithm=mm_hasher_algorithm,

744 mm_shm_cache_max_object_size_mb=mm_shm_cache_max_object_size_mb,

745 mm_encoder_only=mm_encoder_only,

746 mm_encoder_tp_mode=mm_encoder_tp_mode,

vllm/config/multimodal.py

+20 -2

@@ -3,13 +3,13 @@

4 from collections.abc import Mapping

5 from pathlib import Path

6 -from typing import Any, Literal, TypeAlias, TypedDict, final

6 +from typing import Any, Literal, TypeAlias, TypedDict, cast, final

8 from pydantic import ConfigDict, Field, field_validator, model_validator

9 from pydantic.dataclasses import dataclass

11 import vllm.envs as envs

12 -from vllm.config.utils import config

12 +from vllm.config.utils import config, get_from_deprecated_env_if_set

13 from vllm.utils.hashing import safe_hash

14 from vllm.v1.attention.backends.registry import AttentionBackendEnum

@@ -63,6 +63,19 @@ MMEncoderTPMode = Literal["weights", "data"]

63 MMCacheType = Literal["shm", "lru"]

64 VideoPruningMethod = Literal["evs", "vidcom2"]

65 MMTensorIPC = Literal["direct_rpc", "torch_shm"]

66 +MMHasherAlgorithm = Literal["blake3", "sha256", "sha512"]

67 +

68 +

69 +def _get_mm_hasher_algorithm() -> MMHasherAlgorithm:

70 + env_value = get_from_deprecated_env_if_set(

71 + "VLLM_MM_HASHER_ALGORITHM",

72 + "v0.27",

73 + "mm_hasher_algorithm",

74 + )

75 + env_value = "blake3" if env_value is None else env_value

76 + return cast(MMHasherAlgorithm, env_value.lower())

77 +

78 +

79 MMDummyOptions: TypeAlias = dict[str, BaseDummyOptions]

80 """

81 A dictionary containing an entry for each modality type of dummy data.

@@ -134,6 +147,11 @@ class MultiModalConfig:

147 mm_processor_cache_type: MMCacheType = "lru"

148 """Type of cache to use for the multi-modal preprocessor/mapper. If `shm`,

149 use shared memory FIFO cache. If `lru`, use mirrored LRU cache."""

150 + mm_hasher_algorithm: MMHasherAlgorithm = Field(

151 + default_factory=_get_mm_hasher_algorithm

152 + )

153 + """Hash algorithm to use for multi-modal input caching. Use `"sha256"` or

154 + `"sha512"` for FIPS-compliant deployments."""

155 mm_shm_cache_max_object_size_mb: int = Field(default=128, ge=0)

156 """Size limit (in MiB) for each object stored in the multi-modal processor

157 shared memory cache. Only effective when `mm_processor_cache_type` is

vllm/engine/arg_utils.py

+13 -1

@@ -86,7 +86,12 @@ from vllm.config.model import (

86 RunnerOption,

87 TokenizerMode,

88 )

89 -from vllm.config.multimodal import MMCacheType, MMEncoderTPMode, MMTensorIPC

89 +from vllm.config.multimodal import (

90 + MMCacheType,

91 + MMEncoderTPMode,

92 + MMHasherAlgorithm,

93 + MMTensorIPC,

94 +)

95 from vllm.config.observability import DetailedTraceModules

96 from vllm.config.parallel import (

97 All2AllBackend,

@@ -566,6 +571,9 @@ class EngineArgs:

571 mm_processor_cache_type: MMCacheType | None = (

572 MultiModalConfig.mm_processor_cache_type

573 )

574 + mm_hasher_algorithm: MMHasherAlgorithm = get_field(

575 + MultiModalConfig, "mm_hasher_algorithm"

576 + )

577 mm_shm_cache_max_object_size_mb: int = (

578 MultiModalConfig.mm_shm_cache_max_object_size_mb

579 )

@@ -1295,6 +1303,9 @@ class EngineArgs:

1303 multimodal_group.add_argument(

1304 "--mm-processor-cache-type", **multimodal_kwargs["mm_processor_cache_type"]

1305 )

1306 + multimodal_group.add_argument(

1307 + "--mm-hasher-algorithm", **multimodal_kwargs["mm_hasher_algorithm"]

1308 + )

1309 multimodal_group.add_argument(

1310 "--mm-shm-cache-max-object-size-mb",

1311 **multimodal_kwargs["mm_shm_cache_max_object_size_mb"],

@@ -1712,6 +1723,7 @@ class EngineArgs:

1723 mm_processor_cache_gb=self.mm_processor_cache_gb,

1724 mm_processor_cache_type=self.mm_processor_cache_type,

1725 mm_shm_cache_max_object_size_mb=self.mm_shm_cache_max_object_size_mb,

1726 + mm_hasher_algorithm=self.mm_hasher_algorithm,

1727 mm_encoder_only=self.mm_encoder_only,

1728 mm_encoder_tp_mode=self.mm_encoder_tp_mode,

1729 mm_encoder_attn_backend=self.mm_encoder_attn_backend,

vllm/model_executor/models/terratorch.py

+4 -1

@@ -217,7 +217,10 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor[TerratorchProcessing

217 )

218

219 with timing_ctx.record("get_mm_hashes"):

220 - mm_hashes = inputs.get_mm_hashes(self.info.model_id)

220 + mm_hashes = inputs.get_mm_hashes(

221 + self.info.model_id,

222 + self.info.ctx.get_mm_config().mm_hasher_algorithm,

223 + )

224

225 mm_placeholders = {"image": [PlaceholderRange(offset=0, length=0)]}

226

vllm/model_executor/models/transformers/multimodal.py

+4 -1

@@ -402,7 +402,10 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):

402

403 # Use overrides if provided; fallback to data-dependent hashing.

404 with timing_ctx.record("get_mm_hashes"):

405 - mm_hashes = inputs.get_mm_hashes(self.info.model_id)

405 + mm_hashes = inputs.get_mm_hashes(

406 + self.info.model_id,

407 + self.info.ctx.get_mm_config().mm_hasher_algorithm,

408 + )

409

410 # For gemma3 we check `token_type_ids` as the key

411 mm_token_type_ids = processed_data.pop("token_type_ids", None)

vllm/multimodal/hasher.py

+12 -6

@@ -11,7 +11,7 @@ import numpy as np

11 import torch

12 from PIL import Image

14 -import vllm.envs as envs

14 +from vllm.config.multimodal import MMHasherAlgorithm

15 from vllm.logger import init_logger

17 from .media import MediaWithBytes

@@ -20,7 +20,9 @@ logger = init_logger(__name__)

22 @functools.lru_cache(maxsize=3)

23 -def _get_hasher_factory(algorithm: str) -> Callable[[], "hashlib._Hash"]:

23 +def _get_hasher_factory(

24 + algorithm: MMHasherAlgorithm,

25 +) -> Callable[[], "hashlib._Hash"]:

26 """

27 Get the hasher factory based on the configured algorithm.

@@ -32,7 +34,6 @@ def _get_hasher_factory(algorithm: str) -> Callable[[], "hashlib._Hash"]:

35 See: https://github.com/vllm-project/vllm/issues/18334

36 """

35 - algorithm = algorithm.lower()

38 if algorithm == "blake3":

39 from blake3 import blake3

@@ -43,7 +44,7 @@ def _get_hasher_factory(algorithm: str) -> Callable[[], "hashlib._Hash"]:

44 elif algorithm == "sha512":

45 return hashlib.sha512

46 else:

46 - # This should never happen due to env_with_choices validation

47 + # This should never happen due to config validation

48 raise ValueError(f"Unsupported hash algorithm: {algorithm}")

@@ -162,8 +163,13 @@ class MultiModalHasher:

163 yield from cls.serialize_item(obj)

164

165 @classmethod

165 - def hash_kwargs(cls, **kwargs: object) -> str:

166 - hasher_factory = _get_hasher_factory(envs.VLLM_MM_HASHER_ALGORITHM)

166 + def hash_kwargs(

167 + cls,

168 + algorithm: MMHasherAlgorithm,

169 + /,

170 + **kwargs: object,

171 + ) -> str:

172 + hasher_factory = _get_hasher_factory(algorithm)

173 hasher = hasher_factory()

174

175 for k, v in sorted(kwargs.items(), key=lambda kv: kv[0]):

vllm/multimodal/inputs.py

+24 -7

@@ -229,40 +229,57 @@ Uses a list instead of a tensor if the dimensions of each element do not match.

229 """

230

231

232 -def nested_tensors_equal(a: NestedTensors, b: NestedTensors) -> bool:

232 +def nested_tensors_equal(

233 + a: NestedTensors,

234 + b: NestedTensors,

235 + check_dtype: bool = True,

236 +) -> bool:

237 """

238 Equality check between

239 [`NestedTensors`][vllm.multimodal.inputs.NestedTensors] objects.

240 +

241 + If `check_dtype` is `True`, the tensors must have the same dtype.

242 """

243 + check_dtype_func = (

244 + lambda a, b, check_dtype: a.dtype == b.dtype if check_dtype else True

245 + )

246 if isinstance(a, torch.Tensor):

238 - return isinstance(b, torch.Tensor) and torch.equal(a, b)

247 + return (

248 + isinstance(b, torch.Tensor)

249 + and torch.equal(a, b)

250 + and check_dtype_func(a, b, check_dtype)

251 + )

252 elif isinstance(b, torch.Tensor):

240 - return isinstance(a, torch.Tensor) and torch.equal(b, a)

253 + return (

254 + isinstance(a, torch.Tensor)

255 + and torch.equal(b, a)

256 + and check_dtype_func(b, a, check_dtype)

257 + )

258

259 if isinstance(a, list):

260 return (

261 isinstance(b, list)

262 and len(a) == len(b)

246 - and all(nested_tensors_equal(a_, b_) for a_, b_ in zip(a, b))

263 + and all(nested_tensors_equal(a_, b_, check_dtype) for a_, b_ in zip(a, b))

264 )

265 if isinstance(b, list):

266 return (

267 isinstance(a, list)

268 and len(b) == len(a)

252 - and all(nested_tensors_equal(b_, a_) for b_, a_ in zip(b, a))

269 + and all(nested_tensors_equal(b_, a_, check_dtype) for b_, a_ in zip(b, a))

270 )

271

272 if isinstance(a, tuple):

273 return (

274 isinstance(b, tuple)

275 and len(a) == len(b)

259 - and all(nested_tensors_equal(a_, b_) for a_, b_ in zip(a, b))

276 + and all(nested_tensors_equal(a_, b_, check_dtype) for a_, b_ in zip(a, b))

277 )

278 if isinstance(b, tuple):

279 return (

280 isinstance(a, tuple)

281 and len(b) == len(a)

265 - and all(nested_tensors_equal(b_, a_) for b_, a_ in zip(b, a))

282 + and all(nested_tensors_equal(b_, a_, check_dtype) for b_, a_ in zip(b, a))

283 )

284

285 # Both a and b are scalars

vllm/multimodal/processing/inputs.py

+8 -1

@@ -3,6 +3,7 @@

3 from collections.abc import Mapping

4 from dataclasses import dataclass, field

6 +from vllm.config.multimodal import MMHasherAlgorithm

7 from vllm.inputs import MultiModalHashes

9 from ..hasher import MultiModalHasher

@@ -22,7 +23,11 @@ class ProcessorInputs:

23 hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)

24 tokenization_kwargs: Mapping[str, object] = field(default_factory=dict)

25 - def get_mm_hashes(self, model_id: str) -> MultiModalHashes:

26 + def get_mm_hashes(

27 + self,

28 + model_id: str,

29 + hash_algorithm: MMHasherAlgorithm,

30 + ) -> MultiModalHashes:

31 mm_data_items = self.mm_data_items

32 mm_uuid_items = self.mm_uuid_items or {}

33 hf_processor_mm_kwargs = self.hf_processor_mm_kwargs

@@ -49,6 +54,7 @@ class ProcessorInputs:

54 item = uuid_item if uuid_item is not None else item

55 hashes.append(

56 hasher.hash_kwargs(

57 + hash_algorithm,

58 model_id=model_id,

59 **{modality: item},

60 **hf_processor_mm_kwargs,

@@ -61,6 +67,7 @@ class ProcessorInputs:

67 else:

68 mm_hashes[modality] = [

69 hasher.hash_kwargs(

70 + hash_algorithm,

71 model_id=model_id,

72 **{modality: item},

73 **hf_processor_mm_kwargs,

vllm/multimodal/processing/processor.py

+8 -2

@@ -1422,7 +1422,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1422

1423 # Use overrides if provided; fallback to data-dependent hashing.

1424 with timing_ctx.record("get_mm_hashes"):

1425 - mm_hashes = inputs.get_mm_hashes(self.info.model_id)

1425 + mm_hashes = inputs.get_mm_hashes(

1426 + self.info.model_id,

1427 + self.info.ctx.get_mm_config().mm_hasher_algorithm,

1428 + )

1429

1430 mm_prompt_updates = self._get_mm_prompt_updates(

1431 inputs.mm_data_items,

@@ -1454,7 +1457,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1457 return self._apply_hf_processor(inputs, timing_ctx)

1458

1459 with timing_ctx.record("get_mm_hashes"):

1457 - mm_hashes = inputs.get_mm_hashes(self.info.model_id)

1460 + mm_hashes = inputs.get_mm_hashes(

1461 + self.info.model_id,

1462 + self.info.ctx.get_mm_config().mm_hasher_algorithm,

1463 + )

1464

1465 with timing_ctx.record("get_cache_missing_items"):

1466 mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(

vllm/multimodal/utils.py

+31 -19

@@ -16,13 +16,13 @@ from typing_extensions import deprecated

16 from vllm.inputs import MultiModalPlaceholders

17 from vllm.utils.import_utils import LazyLoader

19 -from .hasher import MultiModalHasher

19 from .inputs import (

20 BatchedTensorInputs,

21 MultiModalFeatureSpec,

22 MultiModalFieldElem,

23 MultiModalKwargsItem,

24 MultiModalSharedField,

25 + nested_tensors_equal,

26 )

27 from .media import (

28 AudioMediaIO,

@@ -165,11 +165,28 @@ def argsort_mm_positions(

165 return [(modality, idx) for modality, idx, _ in sorted_flat_items]

166

167

168 -def _get_group_hash(elem: MultiModalFieldElem):

169 - if not isinstance(elem.field, MultiModalSharedField):

170 - return None

168 +def _can_batch_mm_items(

169 + left: MultiModalKwargsItem,

170 + right: MultiModalKwargsItem,

171 +) -> bool:

172 + if left.keys() != right.keys():

173 + return False

174

172 - return MultiModalHasher.hash_kwargs(data=elem.data)

175 + for key, left_elem in left.items():

176 + right_elem = right[key]

177 + left_field, right_field = left_elem.field, right_elem.field

178 + is_shared_field = isinstance(left_field, MultiModalSharedField) and isinstance(

179 + right_field, MultiModalSharedField

180 + )

181 + if (type(left_field) is not type(right_field)) or (

182 + is_shared_field

183 + and not nested_tensors_equal(

184 + left_elem.data, right_elem.data, check_dtype=True

185 + )

186 + ):

187 + return False

188 +

189 + return True

190

191

192 def _batch_mm_items(

@@ -217,26 +234,21 @@ def group_and_batch_mm_items(

234 - `kwargs` is a dictionary of keyword arguments to pass to the model;

235 - `num_items` is the corresponding number of items.

236 """

220 - group_ids = [

221 - tuple(

222 - (key, _get_group_hash(elem))

223 - for key, elem in sorted(item.items(), key=lambda kv: kv[0])

224 - )

225 - for item in items

226 - ]

227 - group_sizes = [sum(1 for _ in group) for _, group in groupby(group_ids)]

228 -

237 start_idx = 0

230 - for group_size in group_sizes:

238 + for end_idx in range(1, len(items) + 1):

239 + if end_idx < len(items) and _can_batch_mm_items(

240 + items[end_idx - 1], items[end_idx]

241 + ):

242 + continue

243 +

244 group_data = _batch_mm_items(

232 - items[start_idx : start_idx + group_size],

245 + items[start_idx:end_idx],

246 device=device,

247 pin_memory=pin_memory,

248 )

249

237 - yield group_size, group_data

238 -

239 - start_idx += group_size

250 + yield end_idx - start_idx, group_data

251 + start_idx = end_idx

252

253 assert start_idx == len(items)

254

vllm/renderers/hf.py

+7 -2

@@ -1276,8 +1276,8 @@ class HfRenderer(BaseRenderer[HfTokenizer]):

1276 embeds_prompt["prompt_embeds"] = full_embeds

1277 embeds_prompt["prompt_is_token_ids"] = is_token_ids_mask

1278

1279 - @staticmethod

1279 def _apply_prompt_embeds_to_engine_input(

1280 + self,

1281 engine_input: MultiModalInput,

1282 prompt_embeds_tensors: list[torch.Tensor],

1283 mm_updates: MultiModalPromptUpdates,

@@ -1300,6 +1300,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):

1300 pe_kwargs_items: list[MultiModalKwargsItem] = []

1301 pe_hashes: list[str] = []

1302 pe_placeholders: list[PlaceholderRange] = []

1303 + mm_config = self.model_config.get_multimodal_config()

1304 for tensor, (start, length) in zip(

1305 prompt_embeds_tensors, positions, strict=True

1306 ):

@@ -1313,7 +1314,11 @@ class HfRenderer(BaseRenderer[HfTokenizer]):

1314 }

1315 )

1316 )

1316 - pe_hashes.append(MultiModalHasher.hash_kwargs(prompt_embeds=tensor))

1317 + pe_hashes.append(

1318 + MultiModalHasher.hash_kwargs(

1319 + mm_config.mm_hasher_algorithm, prompt_embeds=tensor

1320 + )

1321 + )

1322 # `is_embed=None` matches the existing image_embeds-style

1323 # "no encoder, just splice the tensor directly" semantics.

1324 pe_placeholders.append(