| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Regression test for DeepSeek-OCR TensorSchema validation with empty images_crop. |
| 4 | |
| 5 | When using the Gundam preset (BASE_SIZE=1024, IMAGE_SIZE=640, CROP_MODE=True), |
| 6 | images that are small enough to not require cropping produce an empty |
| 7 | images_crop tensor with shape (0, 3, 640, 640). The _parse_and_validate_image_input |
| 8 | method must correctly read image_size from this tensor's shape rather than |
| 9 | falling back to base_size, which would cause a TensorSchema mismatch. |
| 10 | |
| 11 | Run with: |
| 12 | pytest tests/models/multimodal/processing/test_deepseek_ocr.py -v |
| 13 | """ |
| 14 | |
| 15 | from types import SimpleNamespace |
| 16 | |
| 17 | import pytest |
| 18 | import torch |
| 19 | from PIL import Image |
| 20 | from transformers import AutoTokenizer |
| 21 | |
| 22 | from vllm.model_executor.models.deepseek_ocr import ( |
| 23 | DeepseekOCRForCausalLM, |
| 24 | DeepseekOCRImagePixelInputs, |
| 25 | ) |
| 26 | from vllm.model_executor.models.deepseek_ocr2 import DeepseekOCR2ForCausalLM |
| 27 | from vllm.transformers_utils.processors.deepseek_ocr import ( |
| 28 | DeepseekOCRProcessor, |
| 29 | ImageTransform, |
| 30 | ) |
| 31 | |
| 32 | MODEL_ID = "deepseek-ai/DeepSeek-OCR" |
| 33 | |
| 34 | |
| 35 | @pytest.fixture(scope="module") |
| 36 | def processor(): |
| 37 | """Load the DeepseekOCRProcessor with tokenizer from HuggingFace.""" |
| 38 | tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) |
| 39 | return DeepseekOCRProcessor(tokenizer=tokenizer) |
| 40 | |
| 41 | |
| 42 | class TestDeepseekOCREmptyImagesCrop: |
| 43 | """Verify TensorSchema validation handles empty images_crop correctly.""" |
| 44 | |
| 45 | def test_empty_images_crop_small_image(self, processor): |
| 46 | """A small image (<=640px) produces empty images_crop and should |
| 47 | not crash the TensorSchema validation. |
| 48 | |
| 49 | Previously, the code used ``numel() > 0`` to decide whether to read |
| 50 | image_size from the tensor shape. When numel()==0, it fell back to |
| 51 | base_size=1024, mismatching the actual tensor dim of 640. |
| 52 | """ |
| 53 | # Small image: both dims <= IMAGE_SIZE (640) → no crops |
| 54 | small_image = Image.new("RGB", (100, 100), color="red") |
| 55 | |
| 56 | result = processor( |
| 57 | prompt="<image>\nDescribe this image.", |
| 58 | images=[small_image], |
| 59 | ) |
| 60 | |
| 61 | pixel_values = result["pixel_values"] |
| 62 | images_crop = result["images_crop"] |
| 63 | images_spatial_crop = result["images_spatial_crop"] |
| 64 | |
| 65 | # Processor must produce an empty crop tensor for a small image |
| 66 | assert images_crop.shape[0] == 0 |
| 67 | |
| 68 | base_size = pixel_values.shape[-1] |
| 69 | image_size = images_crop.shape[-1] if images_crop is not None else base_size |
| 70 | |
| 71 | # This should NOT raise ValueError |
| 72 | schema = DeepseekOCRImagePixelInputs( |
| 73 | type="pixel_values", |
| 74 | data=pixel_values, |
| 75 | images_crop=images_crop, |
| 76 | images_spatial_crop=images_spatial_crop, |
| 77 | resolve_bindings={ |
| 78 | "base_size": base_size, |
| 79 | "image_size": image_size, |
| 80 | }, |
| 81 | ) |
| 82 | |
| 83 | assert schema.data.shape == (1, 3, 1024, 1024) |
| 84 | assert schema.images_crop.shape == (0, 3, 640, 640) |
| 85 | |
| 86 | def test_populated_images_crop_large_image(self, processor): |
| 87 | """A large image (>640px) produces populated images_crop.""" |
| 88 | # Large image: exceeds IMAGE_SIZE (640) → dynamic crop tiles |
| 89 | large_image = Image.new("RGB", (1200, 800), color="blue") |
| 90 | |
| 91 | result = processor( |
| 92 | prompt="<image>\nDescribe this image.", |
| 93 | images=[large_image], |
| 94 | ) |
| 95 | |
| 96 | pixel_values = result["pixel_values"] |
| 97 | images_crop = result["images_crop"] |
| 98 | images_spatial_crop = result["images_spatial_crop"] |
| 99 | |
| 100 | assert images_crop.shape[0] > 0 |
| 101 | |
| 102 | base_size = pixel_values.shape[-1] |
| 103 | image_size = images_crop.shape[-1] |
| 104 | |
| 105 | schema = DeepseekOCRImagePixelInputs( |
| 106 | type="pixel_values", |
| 107 | data=pixel_values, |
| 108 | images_crop=images_crop, |
| 109 | images_spatial_crop=images_spatial_crop, |
| 110 | resolve_bindings={ |
| 111 | "base_size": base_size, |
| 112 | "image_size": image_size, |
| 113 | }, |
| 114 | ) |
| 115 | |
| 116 | assert schema.data.shape == (1, 3, 1024, 1024) |
| 117 | assert schema.images_crop.shape[-1] == 640 |
| 118 | |
| 119 | def test_mismatched_image_size_raises(self, processor): |
| 120 | """Deliberately wrong image_size binding should still be caught |
| 121 | by TensorSchema validation.""" |
| 122 | small_image = Image.new("RGB", (100, 100), color="green") |
| 123 | |
| 124 | result = processor( |
| 125 | prompt="<image>\nDescribe this image.", |
| 126 | images=[small_image], |
| 127 | ) |
| 128 | |
| 129 | pixel_values = result["pixel_values"] |
| 130 | images_crop = result["images_crop"] |
| 131 | images_spatial_crop = result["images_spatial_crop"] |
| 132 | |
| 133 | with pytest.raises(ValueError, match="images_crop"): |
| 134 | DeepseekOCRImagePixelInputs( |
| 135 | type="pixel_values", |
| 136 | data=pixel_values, |
| 137 | images_crop=images_crop, |
| 138 | images_spatial_crop=images_spatial_crop, |
| 139 | resolve_bindings={ |
| 140 | "base_size": 1024, |
| 141 | "image_size": 1024, # Wrong! Tensor has 640 |
| 142 | }, |
| 143 | ) |
| 144 | |
| 145 | |
| 146 | class TestDeepseekOCRInputValidation: |
| 147 | """Bare ``assert`` used to validate user input here returned HTTP 500 |
| 148 | (AssertionError) instead of HTTP 400 (ValueError) and disappeared under |
| 149 | ``python -O``, silently corrupting the tokenized sequence. See #53850. |
| 150 | """ |
| 151 | |
| 152 | def test_mismatched_image_count_raises_value_error(self, processor): |
| 153 | """Two ``<image>`` tokens with one image must raise ValueError, not |
| 154 | AssertionError (which would yield HTTP 500 and, under ``-O``, silently |
| 155 | drop the middle text segment).""" |
| 156 | image = Image.new("RGB", (100, 100), color="red") |
| 157 | with pytest.raises(ValueError, match="does not match number of images"): |
| 158 | processor(prompt="<image> and <image>", images=[image]) |
| 159 | |
| 160 | def test_missing_prompt_raises_value_error(self, processor): |
| 161 | image = Image.new("RGB", (100, 100), color="red") |
| 162 | with pytest.raises(ValueError, match="prompt and images"): |
| 163 | processor(prompt=None, images=[image]) |
| 164 | |
| 165 | def test_missing_images_raises_value_error(self, processor): |
| 166 | with pytest.raises(ValueError, match="prompt and images"): |
| 167 | processor(prompt="<image>\nDescribe this image.", images=None) |
| 168 | |
| 169 | |
| 170 | def _balanced_normalized_pixels( |
| 171 | batch: int = 1, channels: int = 3, height: int = 64, width: int = 64 |
| 172 | ) -> torch.Tensor: |
| 173 | """Build a pixel tensor whose values cancel to an exact sum of 0. |
| 174 | |
| 175 | Matches the normalized half-black / half-white case from the DeepSeek-OCR |
| 176 | processor (Normalize(0.5, 0.5) maps 0→-1 and 255→+1). |
| 177 | """ |
| 178 | assert width % 2 == 0 |
| 179 | pixel_values = torch.ones(batch, channels, height, width) |
| 180 | pixel_values[..., : width // 2] = -1.0 |
| 181 | assert pixel_values.sum().item() == 0.0 |
| 182 | return pixel_values |
| 183 | |
| 184 | |
| 185 | class TestDeepseekOCRZeroSumPixelsAccepted: |
| 186 | """Zero-sum normalized pixels must still produce validated image inputs. |
| 187 | |
| 188 | Returning None from ``_parse_and_validate_image_input`` makes |
| 189 | ``embed_multimodal`` return None and trips the worker encoder-output |
| 190 | sanity check, killing EngineCore. The zero-sum shortcut is therefore |
| 191 | removed; balanced images must parse like any other valid tensor. |
| 192 | """ |
| 193 | |
| 194 | def test_ocr_accepts_balanced_normalized_pixels(self): |
| 195 | pixel_values = _balanced_normalized_pixels() |
| 196 | images_crop = torch.zeros(0, 3, 40, 40) |
| 197 | images_spatial_crop = torch.tensor([[1, 1]]) |
| 198 | |
| 199 | image_input = DeepseekOCRForCausalLM._parse_and_validate_image_input( |
| 200 | None, |
| 201 | pixel_values=pixel_values, |
| 202 | images_crop=images_crop, |
| 203 | images_spatial_crop=images_spatial_crop, |
| 204 | ) |
| 205 | |
| 206 | assert image_input is not None |
| 207 | assert image_input.data is pixel_values |
| 208 | |
| 209 | def test_ocr2_accepts_balanced_normalized_pixels(self): |
| 210 | pixel_values = _balanced_normalized_pixels() |
| 211 | images_crop = torch.zeros(0, 3, 40, 40) |
| 212 | images_spatial_crop = torch.tensor([[1, 1]]) |
| 213 | model = SimpleNamespace(vision_config=SimpleNamespace(image_size=64)) |
| 214 | |
| 215 | image_input = DeepseekOCR2ForCausalLM._parse_and_validate_image_input( |
| 216 | model, |
| 217 | pixel_values=pixel_values, |
| 218 | images_crop=images_crop, |
| 219 | images_spatial_crop=images_spatial_crop, |
| 220 | ) |
| 221 | |
| 222 | assert image_input is not None |
| 223 | assert image_input.data is pixel_values |
| 224 | |
| 225 | def test_image_transform_half_black_half_white_sums_to_zero(self): |
| 226 | """A 1024×1024 half-black/half-white PNG through ImageTransform |
| 227 | (ToTensor + Normalize(0.5, 0.5)) yields an exact zero sum, and parse |
| 228 | must still accept it. |
| 229 | """ |
| 230 | image = Image.new("RGB", (1024, 1024), (0, 0, 0)) |
| 231 | image.paste(Image.new("RGB", (512, 1024), (255, 255, 255)), (512, 0)) |
| 232 | pixel_values = ImageTransform()(image).unsqueeze(0) |
| 233 | assert pixel_values.sum().item() == 0.0 |
| 234 | |
| 235 | images_crop = torch.zeros(0, 3, 640, 640) |
| 236 | images_spatial_crop = torch.tensor([[1, 1]]) |
| 237 | image_input = DeepseekOCRForCausalLM._parse_and_validate_image_input( |
| 238 | None, |
| 239 | pixel_values=pixel_values, |
| 240 | images_crop=images_crop, |
| 241 | images_spatial_crop=images_spatial_crop, |
| 242 | ) |
| 243 | assert image_input is not None |
| 244 | assert torch.equal(image_input.data, pixel_values) |