Validate scale-out multimodal data before engine handoff (#51898)
Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com> Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com>
Clinton Thomas committed
Sep 6, 2026 at 00:58 UTC
1970f3ed4be7fa8620e4ddc4a12c36a8384cfc27
2 files changed
+100
-3
tests/entrypoints/scale_out/token_in_token_out/test_protocol.py
+43
-1
@@ -9,7 +9,13 @@ fail loudly if the validator semantics ever drift.
9
10
import json
11
12
-from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
12
+import pytest
13
+
14
+from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
15
+ GenerateRequest,
16
+ MultiModalFeatures,
17
+ PlaceholderRangeInfo,
18
+)
19
from vllm.sampling_params import SamplingParams
20
21
@@ -68,3 +74,39 @@ def test_internal_instance_construction_treats_all_as_provided():
74
assert req.is_sampling_param_provided("temperature")
75
# And keys we never touched should also count as provided in this path.
76
assert req.is_sampling_param_provided("top_p")
77
+
78
+
79
+def test_multimodal_features_reject_mismatched_parallel_fields():
80
+ with pytest.raises(ValueError, match="same length"):
81
+ MultiModalFeatures(
82
+ mm_hashes={"image": ["a", "b"]},
83
+ mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=1)]},
84
+ kwargs_data={"image": ["encoded"]},
85
+ )
86
+
87
+
88
+def test_multimodal_features_reject_overlapping_placeholders():
89
+ with pytest.raises(ValueError, match="non-overlapping"):
90
+ MultiModalFeatures(
91
+ mm_hashes={"image": ["a", "b"]},
92
+ mm_placeholders={
93
+ "image": [
94
+ PlaceholderRangeInfo(offset=1, length=2),
95
+ PlaceholderRangeInfo(offset=2, length=2),
96
+ ]
97
+ },
98
+ kwargs_data={"image": ["a", "b"]},
99
+ )
100
+
101
+
102
+def test_generate_request_rejects_placeholder_outside_prompt():
103
+ with pytest.raises(ValueError, match="within the token_ids sequence"):
104
+ GenerateRequest(
105
+ token_ids=[1, 2, 3],
106
+ sampling_params=SamplingParams(),
107
+ features=MultiModalFeatures(
108
+ mm_hashes={"image": ["a"]},
109
+ mm_placeholders={"image": [PlaceholderRangeInfo(offset=2, length=2)]},
110
+ kwargs_data={"image": ["encoded"]},
111
+ ),
112
+ )
vllm/entrypoints/scale_out/token_in_token_out/protocol.py
+57
-2
@@ -33,10 +33,10 @@ from vllm.utils import random_uuid
33
class PlaceholderRangeInfo(BaseModel):
34
"""Serializable placeholder location for a single multi-modal item."""
35
36
- offset: int
36
+ offset: int = Field(ge=0)
37
"""Start index of the placeholder tokens in the prompt."""
38
39
- length: int
39
+ length: int = Field(gt=0)
40
"""Number of placeholder tokens."""
41
42
# TODO: add ``is_embed: list[bool] | None`` once the /generate side
@@ -67,6 +67,47 @@ class MultiModalFeatures(BaseModel):
67
``None`` for metadata-only (cache-hit) responses.
68
"""
69
70
+ @model_validator(mode="after")
71
+ def _validate_parallel_fields(self) -> "MultiModalFeatures":
72
+ modalities = set(self.mm_hashes)
73
+ if set(self.mm_placeholders) != modalities:
74
+ raise ValueError(
75
+ "mm_hashes and mm_placeholders must use the same modalities"
76
+ )
77
+ if self.kwargs_data is not None and set(self.kwargs_data) != modalities:
78
+ raise ValueError("kwargs_data must use the same modalities as mm_hashes")
79
+
80
+ flattened_ranges: list[tuple[int, int]] = []
81
+ for modality in modalities:
82
+ num_hashes = len(self.mm_hashes[modality])
83
+ num_placeholders = len(self.mm_placeholders[modality])
84
+ if num_hashes != num_placeholders:
85
+ raise ValueError(
86
+ f"{modality} mm_hashes and mm_placeholders must have "
87
+ "the same length"
88
+ )
89
+ if (
90
+ self.kwargs_data is not None
91
+ and len(self.kwargs_data[modality]) != num_hashes
92
+ ):
93
+ raise ValueError(
94
+ f"{modality} kwargs_data and mm_hashes must have the same length"
95
+ )
96
+ flattened_ranges.extend(
97
+ (placeholder.offset, placeholder.offset + placeholder.length)
98
+ for placeholder in self.mm_placeholders[modality]
99
+ )
100
+
101
+ flattened_ranges.sort()
102
+ for (offset, end), (next_offset, _) in zip(
103
+ flattened_ranges, flattened_ranges[1:]
104
+ ):
105
+ if next_offset < end:
106
+ raise ValueError(
107
+ "mm_placeholders must be globally non-overlapping and sorted"
108
+ )
109
+ return self
110
+
111
112
class GenerateRequest(BaseModel):
113
request_id: str = Field(
@@ -184,6 +225,20 @@ class GenerateRequest(BaseModel):
225
validate_cache_salt(data.get("cache_salt"))
226
return data
227
228
+ @model_validator(mode="after")
229
+ def _validate_multimodal_feature_bounds(self) -> "GenerateRequest":
230
+ if self.features is None:
231
+ return self
232
+
233
+ prompt_len = len(self.token_ids)
234
+ for ranges in self.features.mm_placeholders.values():
235
+ for placeholder in ranges:
236
+ if placeholder.offset + placeholder.length > prompt_len:
237
+ raise ValueError(
238
+ "mm_placeholders must remain within the token_ids sequence"
239
+ )
240
+ return self
241
+
242
def is_sampling_param_provided(self, name: str) -> bool:
243
"""Whether the caller explicitly set ``sampling_params.<name>``.
244